From 2344882d6b91fc403586f173475b87daaa4ee0b5 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 8 Aug 2025 15:03:54 +0000 Subject: [PATCH 01/17] chore: adds ci/cd related infra content --- .github/.OwlBot.lock.yaml | 17 +++++ .github/.OwlBot.yaml | 22 ++++++ .github/CODEOWNERS | 11 +++ .github/CONTRIBUTING.md | 28 ++++++++ .github/PULL_REQUEST_TEMPLATE.md | 7 ++ .github/release-please.yml | 14 ++++ .github/release-trigger.yml | 2 + .github/workflows/lint.yml | 48 +++++++++++++ ci/run_conditional_tests.sh | 113 ++++++++++++++++++++++++++++++ ci/run_single_test.sh | 114 +++++++++++++++++++++++++++++++ 10 files changed, 376 insertions(+) create mode 100644 .github/.OwlBot.lock.yaml create mode 100644 .github/.OwlBot.yaml create mode 100644 .github/CODEOWNERS create mode 100644 .github/CONTRIBUTING.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/release-please.yml create mode 100644 .github/release-trigger.yml create mode 100644 .github/workflows/lint.yml create mode 100644 ci/run_conditional_tests.sh create mode 100644 ci/run_single_test.sh diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml new file mode 100644 index 000000000..cea9eb68f --- /dev/null +++ b/.github/.OwlBot.lock.yaml @@ -0,0 +1,17 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +docker: + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest + digest: sha256:3b3a31be60853477bc39ed8d9bac162cac3ba083724cecaad54eb81d4e4dae9c +# created: 2025-04-16T22:40:03.123475241Z diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml new file mode 100644 index 000000000..8b142686c --- /dev/null +++ b/.github/.OwlBot.yaml @@ -0,0 +1,22 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest + +deep-remove-regex: + - /owl-bot-staging + +begin-after-commit-hash: f2de93abafa306b2ebadf1d10d947db8bcf2bf15 + diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..6763f258c --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,11 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + +# The @googleapis/api-bigquery is the default owner for changes in this repo +* @googleapis/api-bigquery @googleapis/yoshi-python + +# The python-samples-reviewers team is the default owner for samples changes +/samples/ @googleapis/api-bigquery @googleapis/python-samples-owners @googleapis/yoshi-python diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 000000000..939e5341e --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,28 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows [Google's Open Source Community +Guidelines](https://opensource.google.com/conduct/). diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..65ceeeb5e --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,7 @@ +Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: +- [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea +- [ ] Ensure the tests and linter pass +- [ ] Code coverage does not decrease (if any source code was changed) +- [ ] Appropriate docs were updated (if necessary) + +Fixes # 🦕 diff --git a/.github/release-please.yml b/.github/release-please.yml new file mode 100644 index 000000000..5161ab347 --- /dev/null +++ b/.github/release-please.yml @@ -0,0 +1,14 @@ +releaseType: python +handleGHRelease: true +# NOTE: this section is generated by synthtool.languages.python +# See https://github.com/googleapis/synthtool/blob/master/synthtool/languages/python.py +branches: +- branch: v2 + handleGHRelease: true + releaseType: python +- branch: v1 + handleGHRelease: true + releaseType: python +- branch: v0 + handleGHRelease: true + releaseType: python diff --git a/.github/release-trigger.yml b/.github/release-trigger.yml new file mode 100644 index 000000000..b975c190d --- /dev/null +++ b/.github/release-trigger.yml @@ -0,0 +1,2 @@ +enabled: true +multiScmName: python-bigquery diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..7e8e4c4b2 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,48 @@ +on: + pull_request: + branches: + - main + # Trigger workflow on GitHub merge queue events + # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#merge_group + merge_group: + types: [checks_requested] +name: lint + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` + # See https://github.com/googleapis/google-cloud-python/issues/12013 + # and https://github.com/actions/checkout#checkout-head. + with: + fetch-depth: 2 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run lint + env: + BUILD_TYPE: presubmit + TEST_TYPE: lint + # TODO(https://github.com/googleapis/google-cloud-python/issues/13775): Specify `PY_VERSION` rather than relying on the default python version of the nox session. + PY_VERSION: "unused" + run: | + ci/run_conditional_tests.sh + - name: Run lint_setup_py + env: + BUILD_TYPE: presubmit + TEST_TYPE: lint_setup_py + # TODO(https://github.com/googleapis/google-cloud-python/issues/13775): Specify `PY_VERSION` rather than relying on the default python version of the nox session. + PY_VERSION: "unused" + run: | + ci/run_conditional_tests.sh \ No newline at end of file diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh new file mode 100644 index 000000000..91e34a6de --- /dev/null +++ b/ci/run_conditional_tests.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script requires the following environment variables to be set: +# `BUILD_TYPE` should be one of ["presubmit", "continuous"] +# `TEST_TYPE` should be one of ["docs", "docfx", "prerelease", "unit"] +# or match the name of the nox session that you want to run. +# `PY_VERSION` should be one of ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + +# `TEST_TYPE` and `PY_VERSION` are required by the script `ci/run_single_test.sh` + +# This script will determine which directories have changed +# under the `packages` folder. For `BUILD_TYPE=="presubmit"`, +# we'll compare against the `packages` folder in HEAD, +# whereas for `BUILD_TYPE=="continuous"` we'll compare changes +# with HEAD~1. For all directories that have changed files, we will +# run the script located at `${PROJECT_ROOT}/ci/run_single_test.sh`. + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to non-zero if any command fails, +# or zero if all commands in the pipeline exit successfully. +set -eo pipefail + +export PROJECT_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/..) + +# A script file for running the test in a sub project. +test_script="${PROJECT_ROOT}/ci/run_single_test.sh" + +if [ ${BUILD_TYPE} == "presubmit" ]; then + # For presubmit build, we want to know the difference from the + # common commit in origin/main. + GIT_DIFF_ARG="origin/main..." + + # Then fetch enough history for finding the common commit. + git fetch origin main --deepen=200 + +elif [ ${BUILD_TYPE} == "continuous" ]; then + # For continuous build, we want to know the difference in the last + # commit. This assumes we use squash commit when merging PRs. + GIT_DIFF_ARG="HEAD~.." + + # Then fetch one last commit for getting the diff. + git fetch origin main --deepen=1 + +else + # Run everything. + GIT_DIFF_ARG="" +fi + +# Then detect changes in the test scripts. + +set +e +git diff --quiet ${GIT_DIFF_ARG} ci +changed=$? +set -e + +# Now we have a fixed list, but we can change it to autodetect if +# necessary. + +subdirs=( + packages +) + +RETVAL=0 + +for subdir in ${subdirs[@]}; do + for d in `ls -d ${subdir}/*/`; do + should_test=false + if [ -n "${GIT_DIFF_ARG}" ]; then + echo "checking changes with 'git diff --quiet ${GIT_DIFF_ARG} ${d}'" + set +e + git diff --quiet ${GIT_DIFF_ARG} ${d} + changed=$? + set -e + if [[ "${changed}" -eq 0 ]]; then + echo "no change detected in ${d}, skipping" + else + echo "change detected in ${d}" + should_test=true + fi + else + # If GIT_DIFF_ARG is empty, run all the tests. + should_test=true + fi + if [ "${should_test}" = true ]; then + echo "running test in ${d}" + pushd ${d} + # Temporarily allow failure. + set +e + ${test_script} + ret=$? + set -e + if [ ${ret} -ne 0 ]; then + RETVAL=${ret} + fi + popd + fi + done +done + +exit ${RETVAL} diff --git a/ci/run_single_test.sh b/ci/run_single_test.sh new file mode 100644 index 000000000..ddbe4c435 --- /dev/null +++ b/ci/run_single_test.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script requires the following environment variables to be set: +# `TEST_TYPE` should be one of ["lint", "lint_setup_py", "docs", "docfx", "prerelease"] +# `PY_VERSION` should be one of ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + +# This script is called by the `ci/run_conditional_tests.sh` script. +# A specific `nox` session will be run, depending on the value of +# `TEST_TYPE` and `PY_VERSION`. For example, if `TEST_TYPE` is +# `lint`, the `nox -s lint` session will be run. + + +set -e + +if [ -z "${TEST_TYPE}" ]; then + echo "missing TEST_TYPE env var" + exit 1 +fi + +if [ -z "${PY_VERSION}" ]; then + echo "missing PY_VERSION env var" + exit 1 +fi + +# Don't fail on errors so we can capture all of the output +set +e + +case ${TEST_TYPE} in + docs) + nox -s docs + # This line needs to be directly after `nox -s docs` in order + # for the failure to appear in Github presubmits + retval=$? + # Clean up built docs and python cache after the build process to avoid + # `[Errno 28] No space left on device` + # See https://github.com/googleapis/google-cloud-python/issues/12271 + rm -rf docs/_build + ;; + docfx) + nox -s docfx + # This line needs to be directly after `nox -s docfx` in order + # for the failure to appear in Github presubmits + retval=$? + # Clean up built docs and python cache after the build process to avoid + # `[Errno 28] No space left on device` + # See https://github.com/googleapis/google-cloud-python/issues/12271 + rm -rf docs/_build + ;; + prerelease) + nox -s prerelease_deps-3.13 + retval=$? + ;; + unit) + case ${PY_VERSION} in + "3.7") + nox -s unit-3.7 + retval=$? + ;; + "3.8") + nox -s unit-3.8 + retval=$? + ;; + "3.9") + nox -s unit-3.9 + retval=$? + ;; + "3.10") + nox -s unit-3.10 + retval=$? + ;; + "3.11") + nox -s unit-3.11 + retval=$? + ;; + "3.12") + nox -s unit-3.12 + retval=$? + ;; + "3.13") + nox -s unit-3.13 + retval=$? + ;; + *) + echo "unsupported PY_VERSION" + exit 1 + ;; + esac + ;; + *) + nox -s ${TEST_TYPE} + retval=$? + ;; + esac + +# Clean up `__pycache__` and `.nox` directories to avoid error +# `No space left on device` seen when running tests in Github Actions +find . | grep -E "(__pycache__)" | xargs rm -rf +rm -rf .nox + +exit ${retval} From ddad52801d4a09e92f855bd05bc8f7f9086e767d Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 8 Aug 2025 15:36:52 +0000 Subject: [PATCH 02/17] updates branch for github CI/CD check to autogen --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7e8e4c4b2..ef59351fb 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,7 +1,7 @@ on: pull_request: branches: - - main + - autogen # Trigger workflow on GitHub merge queue events # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#merge_group merge_group: From 1af6db4b451a7495a49c2209b8ab688fc0a35cf6 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 8 Aug 2025 16:05:13 +0000 Subject: [PATCH 03/17] update run commands --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ef59351fb..ddfef53f3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -37,7 +37,7 @@ jobs: # TODO(https://github.com/googleapis/google-cloud-python/issues/13775): Specify `PY_VERSION` rather than relying on the default python version of the nox session. PY_VERSION: "unused" run: | - ci/run_conditional_tests.sh + nox -s lint - name: Run lint_setup_py env: BUILD_TYPE: presubmit @@ -45,4 +45,4 @@ jobs: # TODO(https://github.com/googleapis/google-cloud-python/issues/13775): Specify `PY_VERSION` rather than relying on the default python version of the nox session. PY_VERSION: "unused" run: | - ci/run_conditional_tests.sh \ No newline at end of file + nox -s lint_setup_py \ No newline at end of file From bc267ce9a0e66267718d83c9f18b6bd3b9680802 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 8 Aug 2025 16:59:14 +0000 Subject: [PATCH 04/17] updates formatting in pagers.py files to pass blacken --- google/cloud/bigquery_v2/services/dataset_service/pagers.py | 1 - google/cloud/bigquery_v2/services/job_service/pagers.py | 1 - google/cloud/bigquery_v2/services/model_service/pagers.py | 1 - google/cloud/bigquery_v2/services/routine_service/pagers.py | 1 - .../bigquery_v2/services/row_access_policy_service/pagers.py | 1 - google/cloud/bigquery_v2/services/table_service/pagers.py | 1 - 6 files changed, 6 deletions(-) diff --git a/google/cloud/bigquery_v2/services/dataset_service/pagers.py b/google/cloud/bigquery_v2/services/dataset_service/pagers.py index c94c0b76a..de0afd87f 100644 --- a/google/cloud/bigquery_v2/services/dataset_service/pagers.py +++ b/google/cloud/bigquery_v2/services/dataset_service/pagers.py @@ -67,7 +67,6 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ): """Instantiate the pager. diff --git a/google/cloud/bigquery_v2/services/job_service/pagers.py b/google/cloud/bigquery_v2/services/job_service/pagers.py index 83f723917..b976c1d75 100644 --- a/google/cloud/bigquery_v2/services/job_service/pagers.py +++ b/google/cloud/bigquery_v2/services/job_service/pagers.py @@ -67,7 +67,6 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ): """Instantiate the pager. diff --git a/google/cloud/bigquery_v2/services/model_service/pagers.py b/google/cloud/bigquery_v2/services/model_service/pagers.py index 567324a11..2a3fc6233 100644 --- a/google/cloud/bigquery_v2/services/model_service/pagers.py +++ b/google/cloud/bigquery_v2/services/model_service/pagers.py @@ -67,7 +67,6 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ): """Instantiate the pager. diff --git a/google/cloud/bigquery_v2/services/routine_service/pagers.py b/google/cloud/bigquery_v2/services/routine_service/pagers.py index 7a67a9cd5..807f01503 100644 --- a/google/cloud/bigquery_v2/services/routine_service/pagers.py +++ b/google/cloud/bigquery_v2/services/routine_service/pagers.py @@ -67,7 +67,6 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ): """Instantiate the pager. diff --git a/google/cloud/bigquery_v2/services/row_access_policy_service/pagers.py b/google/cloud/bigquery_v2/services/row_access_policy_service/pagers.py index 13a621b23..0b7d99f81 100644 --- a/google/cloud/bigquery_v2/services/row_access_policy_service/pagers.py +++ b/google/cloud/bigquery_v2/services/row_access_policy_service/pagers.py @@ -67,7 +67,6 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ): """Instantiate the pager. diff --git a/google/cloud/bigquery_v2/services/table_service/pagers.py b/google/cloud/bigquery_v2/services/table_service/pagers.py index 1555d74c0..108e79eff 100644 --- a/google/cloud/bigquery_v2/services/table_service/pagers.py +++ b/google/cloud/bigquery_v2/services/table_service/pagers.py @@ -67,7 +67,6 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ): """Instantiate the pager. From c656ee933f8ce0b9c47aa7a33f72a7675ce8dd0f Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 8 Aug 2025 17:01:29 +0000 Subject: [PATCH 05/17] removes unused check --- .github/workflows/lint.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ddfef53f3..eea31b494 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -38,11 +38,3 @@ jobs: PY_VERSION: "unused" run: | nox -s lint - - name: Run lint_setup_py - env: - BUILD_TYPE: presubmit - TEST_TYPE: lint_setup_py - # TODO(https://github.com/googleapis/google-cloud-python/issues/13775): Specify `PY_VERSION` rather than relying on the default python version of the nox session. - PY_VERSION: "unused" - run: | - nox -s lint_setup_py \ No newline at end of file From af7b6568c9571f32030bf1b57f0cfdc5a2531ac5 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 8 Aug 2025 18:17:38 +0000 Subject: [PATCH 06/17] adds unittest.yml --- .github/workflows/unittest.yml | 118 +++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 .github/workflows/unittest.yml diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml new file mode 100644 index 000000000..7d84a6223 --- /dev/null +++ b/.github/workflows/unittest.yml @@ -0,0 +1,118 @@ +on: + pull_request: + branches: + - autogen + # Trigger workflow on GitHub merge queue events + # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#merge_group + merge_group: + types: [checks_requested] +name: unittest + +permissions: + contents: read + +jobs: + unit: + # TODO(https://github.com/googleapis/gapic-generator-python/issues/2303): use `ubuntu-latest` once this bug is fixed. + # Use ubuntu-22.04 until Python 3.7 is removed from the test matrix + # https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories + runs-on: ubuntu-22.04 + strategy: + matrix: + python: ['3.9', '3.10', "3.11", "3.12", "3.13"] + steps: + - name: Checkout + uses: actions/checkout@v4 + # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` + # See https://github.com/googleapis/google-cloud-python/issues/12013 + # and https://github.com/actions/checkout#checkout-head. + with: + fetch-depth: 2 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run unit tests + env: + COVERAGE_FILE: .coverage-${{ matrix.python }} + BUILD_TYPE: presubmit + TEST_TYPE: unit + PY_VERSION: ${{ matrix.python }} + run: | + nox -s unit-${{ matrix.python }} + - name: Upload coverage results + uses: actions/upload-artifact@v4 + with: + name: coverage-artifact-${{ '{{' }} matrix.python {{ '}}' }} + path: .coverage-${{ matrix.python }} + unit-extended: + name: ${{ matrix.option }} + runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.13"] + option: ["prerelease_deps"] + steps: + - name: Checkout + uses: actions/checkout@v4 + # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` + # See https://github.com/googleapis/google-cloud-python/issues/12013 + # and https://github.com/actions/checkout#checkout-head. + with: + fetch-depth: 2 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run ${{ matrix.option }} tests + env: + BUILD_TYPE: presubmit + TEST_TYPE: ${{ matrix.option }} + # TODO(https://github.com/googleapis/google-cloud-python/issues/13775): Specify `PY_VERSION` rather than relying on the default python version of the nox session. + PY_VERSION: "unused" + run: | + nox -s unit-${{ matrix.option }}-${{ matrix.python }} + + cover: + runs-on: ubuntu-latest + needs: + - unit + steps: + - name: Checkout + uses: actions/checkout@v4 + # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` + # See https://github.com/googleapis/google-cloud-python/issues/12013 + # and https://github.com/actions/checkout#checkout-head. + with: + fetch-depth: 2 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Set number of files changes in packages directory + id: packages + run: echo "::set-output name=num_files_changed::$(git diff HEAD~1 -- packages | wc -l)" + - name: Install coverage + if: steps.packages.num_files_changed > 0 + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install coverage + - name: Download coverage results + if: ${{ steps.date.packages.num_files_changed > 0 }} + uses: actions/download-artifact@v4 + with: + path: .coverage-results/ + - name: Report coverage results + if: ${{ steps.date.packages.num_files_changed > 0 }} + run: | + find .coverage-results -type f -name '*.zip' -exec unzip {} \; + coverage combine .coverage-results/**/.coverage* + coverage report --show-missing --fail-under=100 \ No newline at end of file From f926a08da92953669e9dad19c67b51a98f7ce523 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 8 Aug 2025 18:21:00 +0000 Subject: [PATCH 07/17] adds removes matrix.options --- .github/workflows/unittest.yml | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 7d84a6223..4f118eca0 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -49,37 +49,6 @@ jobs: with: name: coverage-artifact-${{ '{{' }} matrix.python {{ '}}' }} path: .coverage-${{ matrix.python }} - unit-extended: - name: ${{ matrix.option }} - runs-on: ubuntu-latest - strategy: - matrix: - python: ["3.13"] - option: ["prerelease_deps"] - steps: - - name: Checkout - uses: actions/checkout@v4 - # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` - # See https://github.com/googleapis/google-cloud-python/issues/12013 - # and https://github.com/actions/checkout#checkout-head. - with: - fetch-depth: 2 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - - name: Install nox - run: | - python -m pip install --upgrade setuptools pip wheel - python -m pip install nox - - name: Run ${{ matrix.option }} tests - env: - BUILD_TYPE: presubmit - TEST_TYPE: ${{ matrix.option }} - # TODO(https://github.com/googleapis/google-cloud-python/issues/13775): Specify `PY_VERSION` rather than relying on the default python version of the nox session. - PY_VERSION: "unused" - run: | - nox -s unit-${{ matrix.option }}-${{ matrix.python }} cover: runs-on: ubuntu-latest From bcb3d5c4dfe06d44a1ecc5904cc27b5655d8334a Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 8 Aug 2025 18:42:35 +0000 Subject: [PATCH 08/17] adds kokoro linting-typing.cfg --- .kokoro/presubmit/linting-typing.cfg | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .kokoro/presubmit/linting-typing.cfg diff --git a/.kokoro/presubmit/linting-typing.cfg b/.kokoro/presubmit/linting-typing.cfg new file mode 100644 index 000000000..23654b90c --- /dev/null +++ b/.kokoro/presubmit/linting-typing.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run these nox sessions. +env_vars: { + key: "NOX_SESSION" + value: "" +} \ No newline at end of file From 27ad7b7616ea550b5402600a2f2c96d90ee4dc4d Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 8 Aug 2025 18:45:06 +0000 Subject: [PATCH 09/17] mods kokoro linting-typing.cfg --- .kokoro/presubmit/linting-typing.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/presubmit/linting-typing.cfg b/.kokoro/presubmit/linting-typing.cfg index 23654b90c..219c220a4 100644 --- a/.kokoro/presubmit/linting-typing.cfg +++ b/.kokoro/presubmit/linting-typing.cfg @@ -3,5 +3,5 @@ # Only run these nox sessions. env_vars: { key: "NOX_SESSION" - value: "" + value: "unit-3.13" } \ No newline at end of file From c362a5f05a9b8fd9fca3557ede77795cd7d5f2bd Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 8 Aug 2025 18:47:45 +0000 Subject: [PATCH 10/17] mods kokoro linting-typing.cfg again --- .kokoro/presubmit/linting-typing.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/presubmit/linting-typing.cfg b/.kokoro/presubmit/linting-typing.cfg index 219c220a4..0c2c3478c 100644 --- a/.kokoro/presubmit/linting-typing.cfg +++ b/.kokoro/presubmit/linting-typing.cfg @@ -3,5 +3,5 @@ # Only run these nox sessions. env_vars: { key: "NOX_SESSION" - value: "unit-3.13" + value: "lint" } \ No newline at end of file From a22f3c6f8704c030dca0d15ae4ceeb4e9786fb06 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 8 Aug 2025 18:54:24 +0000 Subject: [PATCH 11/17] adds kokoro .sh files --- .kokoro/build.sh | 60 ++++ .kokoro/populate-secrets.sh | 43 +++ .kokoro/test-samples-against-head.sh | 26 ++ .kokoro/test-samples-impl.sh | 102 ++++++ .kokoro/test-samples.sh | 44 +++ .kokoro/trampoline.sh | 28 ++ .kokoro/trampoline_v2.sh | 487 +++++++++++++++++++++++++++ 7 files changed, 790 insertions(+) create mode 100755 .kokoro/build.sh create mode 100755 .kokoro/populate-secrets.sh create mode 100755 .kokoro/test-samples-against-head.sh create mode 100755 .kokoro/test-samples-impl.sh create mode 100755 .kokoro/test-samples.sh create mode 100755 .kokoro/trampoline.sh create mode 100755 .kokoro/trampoline_v2.sh diff --git a/.kokoro/build.sh b/.kokoro/build.sh new file mode 100755 index 000000000..d41b45aa1 --- /dev/null +++ b/.kokoro/build.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") + +if [[ -z "${PROJECT_ROOT:-}" ]]; then + PROJECT_ROOT=$(realpath "${CURRENT_DIR}/..") +fi + +pushd "${PROJECT_ROOT}" + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Setup service account credentials. +if [[ -f "${KOKORO_GFILE_DIR}/service-account.json" ]] +then + export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json +fi + +# Setup project id. +if [[ -f "${KOKORO_GFILE_DIR}/project-id.json" ]] +then + export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") +fi + +# If this is a continuous build, send the test log to the FlakyBot. +# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then + cleanup() { + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + } + trap cleanup EXIT HUP +fi + +# If NOX_SESSION is set, it only runs the specified session, +# otherwise run all the sessions. +if [[ -n "${NOX_SESSION:-}" ]]; then + python3 -m nox -s ${NOX_SESSION:-} +else + python3 -m nox +fi diff --git a/.kokoro/populate-secrets.sh b/.kokoro/populate-secrets.sh new file mode 100755 index 000000000..c435402f4 --- /dev/null +++ b/.kokoro/populate-secrets.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2024 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;} +function msg { println "$*" >&2 ;} +function println { printf '%s\n' "$(now) $*" ;} + + +# Populates requested secrets set in SECRET_MANAGER_KEYS from service account: +# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com +SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" +msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" +mkdir -p ${SECRET_LOCATION} +for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") +do + msg "Retrieving secret ${key}" + docker run --entrypoint=gcloud \ + --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \ + gcr.io/google.com/cloudsdktool/cloud-sdk \ + secrets versions access latest \ + --project cloud-devrel-kokoro-resources \ + --secret ${key} > \ + "${SECRET_LOCATION}/${key}" + if [[ $? == 0 ]]; then + msg "Secret written to ${SECRET_LOCATION}/${key}" + else + msg "Error retrieving secret ${key}" + fi +done diff --git a/.kokoro/test-samples-against-head.sh b/.kokoro/test-samples-against-head.sh new file mode 100755 index 000000000..e9d8bd79a --- /dev/null +++ b/.kokoro/test-samples-against-head.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A customized test runner for samples. +# +# For periodic builds, you can specify this file for testing against head. + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh new file mode 100755 index 000000000..40e248822 --- /dev/null +++ b/.kokoro/test-samples-impl.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +# Exit early if samples don't exist +if ! find samples -name 'requirements.txt' | grep -q .; then + echo "No tests run. './samples/**/requirements.txt' not found" + exit 0 +fi + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Install nox +python3.9 -m pip install --upgrade --quiet nox virtualenv + +# Use secrets acessor service account to get secrets +if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then + gcloud auth activate-service-account \ + --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ + --project="cloud-devrel-kokoro-resources" +fi + +# This script will create 3 files: +# - testing/test-env.sh +# - testing/service-account.json +# - testing/client-secrets.json +./scripts/decrypt-secrets.sh + +source ./testing/test-env.sh +export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json + +# For cloud-run session, we activate the service account for gcloud sdk. +gcloud auth activate-service-account \ + --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" + +export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json + +echo -e "\n******************** TESTING PROJECTS ********************" + +# Switch to 'fail at end' to allow all tests to complete before exiting. +set +e +# Use RTN to return a non-zero value if the test fails. +RTN=0 +ROOT=$(pwd) +# Find all requirements.txt in the samples directory (may break on whitespace). +for file in samples/**/requirements.txt; do + cd "$ROOT" + # Navigate to the project folder. + file=$(dirname "$file") + cd "$file" + + echo "------------------------------------------------------------" + echo "- testing $file" + echo "------------------------------------------------------------" + + # Use nox to execute the tests for the project. + python3.9 -m nox -s "$RUN_TESTS_SESSION" + EXIT=$? + + # If this is a periodic build, send the test log to the FlakyBot. + # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. + if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + fi + + if [[ $EXIT -ne 0 ]]; then + RTN=1 + echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" + else + echo -e "\n Testing completed.\n" + fi + +done +cd "$ROOT" + +# Workaround for Kokoro permissions issue: delete secrets +rm testing/{test-env.sh,client-secrets.json,service-account.json} + +exit "$RTN" diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh new file mode 100755 index 000000000..7933d8201 --- /dev/null +++ b/.kokoro/test-samples.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The default test runner for samples. +# +# For periodic builds, we rewinds the repo to the latest release, and +# run test-samples-impl.sh. + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +# Run periodic samples tests at latest release +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + # preserving the test runner implementation. + cp .kokoro/test-samples-impl.sh "${TMPDIR}/test-samples-impl.sh" + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + echo "Now we rewind the repo back to the latest release..." + LATEST_RELEASE=$(git describe --abbrev=0 --tags) + git checkout $LATEST_RELEASE + echo "The current head is: " + echo $(git rev-parse --verify HEAD) + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + # move back the test runner implementation if there's no file. + if [ ! -f .kokoro/test-samples-impl.sh ]; then + cp "${TMPDIR}/test-samples-impl.sh" .kokoro/test-samples-impl.sh + fi +fi + +exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh new file mode 100755 index 000000000..48f796997 --- /dev/null +++ b/.kokoro/trampoline.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Always run the cleanup script, regardless of the success of bouncing into +# the container. +function cleanup() { + chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + echo "cleanup"; +} +trap cleanup EXIT + +$(dirname $0)/populate-secrets.sh # Secret Manager secrets. +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" \ No newline at end of file diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh new file mode 100755 index 000000000..35fa52923 --- /dev/null +++ b/.kokoro/trampoline_v2.sh @@ -0,0 +1,487 @@ +#!/usr/bin/env bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# To run this script, first download few files from gcs to /dev/shm. +# (/dev/shm is passed into the container as KOKORO_GFILE_DIR). +# +# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm +# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm +# +# Then run the script. +# .kokoro/trampoline_v2.sh +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. + + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.5" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For FlakyBot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" +else + PROJECT_ROOT="$(repo_root $(pwd))" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}:latest" | grep "${TRAMPOLINE_IMAGE}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" From d180c02e2ac298d579b6877e7d8509838ae35ea3 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 11 Aug 2025 19:46:37 +0000 Subject: [PATCH 12/17] comments out a small snippet for debugging --- .../services/centralized_services/client.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/google/cloud/bigquery_v2/services/centralized_services/client.py b/google/cloud/bigquery_v2/services/centralized_services/client.py index aa581d2ab..ba2a04dfa 100644 --- a/google/cloud/bigquery_v2/services/centralized_services/client.py +++ b/google/cloud/bigquery_v2/services/centralized_services/client.py @@ -164,17 +164,17 @@ def list_models( # Sample TODO: Relocate this to a samples file # =============================================== -# Instantiate BQClient class -bqclient = BigQueryClient() +# # Instantiate BQClient class +# bqclient = BigQueryClient() -# Instantiate Request class -get_dataset_request = GetDatasetRequest( - project_id=PROJECT_ID, - dataset_id="experimental", -) +# # Instantiate Request class +# get_dataset_request = GetDatasetRequest( +# project_id=PROJECT_ID, +# dataset_id="experimental", +# ) -# Generate response -dataset = bqclient.get_dataset(request=get_dataset_request) +# # Generate response +# dataset = bqclient.get_dataset(request=get_dataset_request) -# Display response -print(f"GET DATASET:\n\t{dataset.id=}\n") +# # Display response +# print(f"GET DATASET:\n\t{dataset.id=}\n") From 29f1ab792079cfe642c0406261de42892dbceeb4 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Mon, 11 Aug 2025 16:12:50 -0400 Subject: [PATCH 13/17] Update google/cloud/bigquery_v2/services/job_service/pagers.py --- google/cloud/bigquery_v2/services/job_service/pagers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/google/cloud/bigquery_v2/services/job_service/pagers.py b/google/cloud/bigquery_v2/services/job_service/pagers.py index b976c1d75..83f723917 100644 --- a/google/cloud/bigquery_v2/services/job_service/pagers.py +++ b/google/cloud/bigquery_v2/services/job_service/pagers.py @@ -67,6 +67,7 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), + ): """Instantiate the pager. From f295759646b878319aaf9c6da5697d481d1c0696 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Mon, 11 Aug 2025 16:15:07 -0400 Subject: [PATCH 14/17] Update .github/workflows/lint.yml --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index eea31b494..d06fd2a7c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -37,4 +37,4 @@ jobs: # TODO(https://github.com/googleapis/google-cloud-python/issues/13775): Specify `PY_VERSION` rather than relying on the default python version of the nox session. PY_VERSION: "unused" run: | - nox -s lint + nox -s blacken lint From 799cd64bcdf0a1f2a3e1f04e933597f1a73c1071 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 11 Aug 2025 20:18:27 +0000 Subject: [PATCH 15/17] removes sample code --- .../services/centralized_services/client.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/google/cloud/bigquery_v2/services/centralized_services/client.py b/google/cloud/bigquery_v2/services/centralized_services/client.py index ba2a04dfa..63c403dcb 100644 --- a/google/cloud/bigquery_v2/services/centralized_services/client.py +++ b/google/cloud/bigquery_v2/services/centralized_services/client.py @@ -158,23 +158,3 @@ def list_models( """ kwargs = _drop_self_key(locals()) return self.model_service_client.list_models(**kwargs) - - -# =============================================== -# Sample TODO: Relocate this to a samples file -# =============================================== - -# # Instantiate BQClient class -# bqclient = BigQueryClient() - -# # Instantiate Request class -# get_dataset_request = GetDatasetRequest( -# project_id=PROJECT_ID, -# dataset_id="experimental", -# ) - -# # Generate response -# dataset = bqclient.get_dataset(request=get_dataset_request) - -# # Display response -# print(f"GET DATASET:\n\t{dataset.id=}\n") From fbc2e13f18229e784e6ebbfd63a85092ab43fc4a Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 11 Aug 2025 20:21:27 +0000 Subject: [PATCH 16/17] removes two currently unused ci shell scripts --- ci/run_conditional_tests.sh | 113 ----------------------------------- ci/run_single_test.sh | 114 ------------------------------------ 2 files changed, 227 deletions(-) delete mode 100644 ci/run_conditional_tests.sh delete mode 100644 ci/run_single_test.sh diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh deleted file mode 100644 index 91e34a6de..000000000 --- a/ci/run_conditional_tests.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/bash -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script requires the following environment variables to be set: -# `BUILD_TYPE` should be one of ["presubmit", "continuous"] -# `TEST_TYPE` should be one of ["docs", "docfx", "prerelease", "unit"] -# or match the name of the nox session that you want to run. -# `PY_VERSION` should be one of ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] - -# `TEST_TYPE` and `PY_VERSION` are required by the script `ci/run_single_test.sh` - -# This script will determine which directories have changed -# under the `packages` folder. For `BUILD_TYPE=="presubmit"`, -# we'll compare against the `packages` folder in HEAD, -# whereas for `BUILD_TYPE=="continuous"` we'll compare changes -# with HEAD~1. For all directories that have changed files, we will -# run the script located at `${PROJECT_ROOT}/ci/run_single_test.sh`. - -# `-e` enables the script to automatically fail when a command fails -# `-o pipefail` sets the exit code to non-zero if any command fails, -# or zero if all commands in the pipeline exit successfully. -set -eo pipefail - -export PROJECT_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/..) - -# A script file for running the test in a sub project. -test_script="${PROJECT_ROOT}/ci/run_single_test.sh" - -if [ ${BUILD_TYPE} == "presubmit" ]; then - # For presubmit build, we want to know the difference from the - # common commit in origin/main. - GIT_DIFF_ARG="origin/main..." - - # Then fetch enough history for finding the common commit. - git fetch origin main --deepen=200 - -elif [ ${BUILD_TYPE} == "continuous" ]; then - # For continuous build, we want to know the difference in the last - # commit. This assumes we use squash commit when merging PRs. - GIT_DIFF_ARG="HEAD~.." - - # Then fetch one last commit for getting the diff. - git fetch origin main --deepen=1 - -else - # Run everything. - GIT_DIFF_ARG="" -fi - -# Then detect changes in the test scripts. - -set +e -git diff --quiet ${GIT_DIFF_ARG} ci -changed=$? -set -e - -# Now we have a fixed list, but we can change it to autodetect if -# necessary. - -subdirs=( - packages -) - -RETVAL=0 - -for subdir in ${subdirs[@]}; do - for d in `ls -d ${subdir}/*/`; do - should_test=false - if [ -n "${GIT_DIFF_ARG}" ]; then - echo "checking changes with 'git diff --quiet ${GIT_DIFF_ARG} ${d}'" - set +e - git diff --quiet ${GIT_DIFF_ARG} ${d} - changed=$? - set -e - if [[ "${changed}" -eq 0 ]]; then - echo "no change detected in ${d}, skipping" - else - echo "change detected in ${d}" - should_test=true - fi - else - # If GIT_DIFF_ARG is empty, run all the tests. - should_test=true - fi - if [ "${should_test}" = true ]; then - echo "running test in ${d}" - pushd ${d} - # Temporarily allow failure. - set +e - ${test_script} - ret=$? - set -e - if [ ${ret} -ne 0 ]; then - RETVAL=${ret} - fi - popd - fi - done -done - -exit ${RETVAL} diff --git a/ci/run_single_test.sh b/ci/run_single_test.sh deleted file mode 100644 index ddbe4c435..000000000 --- a/ci/run_single_test.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script requires the following environment variables to be set: -# `TEST_TYPE` should be one of ["lint", "lint_setup_py", "docs", "docfx", "prerelease"] -# `PY_VERSION` should be one of ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] - -# This script is called by the `ci/run_conditional_tests.sh` script. -# A specific `nox` session will be run, depending on the value of -# `TEST_TYPE` and `PY_VERSION`. For example, if `TEST_TYPE` is -# `lint`, the `nox -s lint` session will be run. - - -set -e - -if [ -z "${TEST_TYPE}" ]; then - echo "missing TEST_TYPE env var" - exit 1 -fi - -if [ -z "${PY_VERSION}" ]; then - echo "missing PY_VERSION env var" - exit 1 -fi - -# Don't fail on errors so we can capture all of the output -set +e - -case ${TEST_TYPE} in - docs) - nox -s docs - # This line needs to be directly after `nox -s docs` in order - # for the failure to appear in Github presubmits - retval=$? - # Clean up built docs and python cache after the build process to avoid - # `[Errno 28] No space left on device` - # See https://github.com/googleapis/google-cloud-python/issues/12271 - rm -rf docs/_build - ;; - docfx) - nox -s docfx - # This line needs to be directly after `nox -s docfx` in order - # for the failure to appear in Github presubmits - retval=$? - # Clean up built docs and python cache after the build process to avoid - # `[Errno 28] No space left on device` - # See https://github.com/googleapis/google-cloud-python/issues/12271 - rm -rf docs/_build - ;; - prerelease) - nox -s prerelease_deps-3.13 - retval=$? - ;; - unit) - case ${PY_VERSION} in - "3.7") - nox -s unit-3.7 - retval=$? - ;; - "3.8") - nox -s unit-3.8 - retval=$? - ;; - "3.9") - nox -s unit-3.9 - retval=$? - ;; - "3.10") - nox -s unit-3.10 - retval=$? - ;; - "3.11") - nox -s unit-3.11 - retval=$? - ;; - "3.12") - nox -s unit-3.12 - retval=$? - ;; - "3.13") - nox -s unit-3.13 - retval=$? - ;; - *) - echo "unsupported PY_VERSION" - exit 1 - ;; - esac - ;; - *) - nox -s ${TEST_TYPE} - retval=$? - ;; - esac - -# Clean up `__pycache__` and `.nox` directories to avoid error -# `No space left on device` seen when running tests in Github Actions -find . | grep -E "(__pycache__)" | xargs rm -rf -rm -rf .nox - -exit ${retval} From 74991de64ae7f71147bd0a38e638be755efe9838 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 11 Aug 2025 20:24:54 +0000 Subject: [PATCH 17/17] removes .kokoro files for now --- .kokoro/build.sh | 60 ---- .kokoro/populate-secrets.sh | 43 --- .kokoro/presubmit/linting-typing.cfg | 7 - .kokoro/test-samples-against-head.sh | 26 -- .kokoro/test-samples-impl.sh | 102 ------ .kokoro/test-samples.sh | 44 --- .kokoro/trampoline.sh | 28 -- .kokoro/trampoline_v2.sh | 487 --------------------------- 8 files changed, 797 deletions(-) delete mode 100755 .kokoro/build.sh delete mode 100755 .kokoro/populate-secrets.sh delete mode 100644 .kokoro/presubmit/linting-typing.cfg delete mode 100755 .kokoro/test-samples-against-head.sh delete mode 100755 .kokoro/test-samples-impl.sh delete mode 100755 .kokoro/test-samples.sh delete mode 100755 .kokoro/trampoline.sh delete mode 100755 .kokoro/trampoline_v2.sh diff --git a/.kokoro/build.sh b/.kokoro/build.sh deleted file mode 100755 index d41b45aa1..000000000 --- a/.kokoro/build.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") - -if [[ -z "${PROJECT_ROOT:-}" ]]; then - PROJECT_ROOT=$(realpath "${CURRENT_DIR}/..") -fi - -pushd "${PROJECT_ROOT}" - -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 - -# Debug: show build environment -env | grep KOKORO - -# Setup service account credentials. -if [[ -f "${KOKORO_GFILE_DIR}/service-account.json" ]] -then - export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json -fi - -# Setup project id. -if [[ -f "${KOKORO_GFILE_DIR}/project-id.json" ]] -then - export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") -fi - -# If this is a continuous build, send the test log to the FlakyBot. -# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. -if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then - cleanup() { - chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot - $KOKORO_GFILE_DIR/linux_amd64/flakybot - } - trap cleanup EXIT HUP -fi - -# If NOX_SESSION is set, it only runs the specified session, -# otherwise run all the sessions. -if [[ -n "${NOX_SESSION:-}" ]]; then - python3 -m nox -s ${NOX_SESSION:-} -else - python3 -m nox -fi diff --git a/.kokoro/populate-secrets.sh b/.kokoro/populate-secrets.sh deleted file mode 100755 index c435402f4..000000000 --- a/.kokoro/populate-secrets.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;} -function msg { println "$*" >&2 ;} -function println { printf '%s\n' "$(now) $*" ;} - - -# Populates requested secrets set in SECRET_MANAGER_KEYS from service account: -# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com -SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" -msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" -mkdir -p ${SECRET_LOCATION} -for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") -do - msg "Retrieving secret ${key}" - docker run --entrypoint=gcloud \ - --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \ - gcr.io/google.com/cloudsdktool/cloud-sdk \ - secrets versions access latest \ - --project cloud-devrel-kokoro-resources \ - --secret ${key} > \ - "${SECRET_LOCATION}/${key}" - if [[ $? == 0 ]]; then - msg "Secret written to ${SECRET_LOCATION}/${key}" - else - msg "Error retrieving secret ${key}" - fi -done diff --git a/.kokoro/presubmit/linting-typing.cfg b/.kokoro/presubmit/linting-typing.cfg deleted file mode 100644 index 0c2c3478c..000000000 --- a/.kokoro/presubmit/linting-typing.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run these nox sessions. -env_vars: { - key: "NOX_SESSION" - value: "lint" -} \ No newline at end of file diff --git a/.kokoro/test-samples-against-head.sh b/.kokoro/test-samples-against-head.sh deleted file mode 100755 index e9d8bd79a..000000000 --- a/.kokoro/test-samples-against-head.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A customized test runner for samples. -# -# For periodic builds, you can specify this file for testing against head. - -# `-e` enables the script to automatically fail when a command fails -# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero -set -eo pipefail -# Enables `**` to include files nested inside sub-folders -shopt -s globstar - -exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh deleted file mode 100755 index 40e248822..000000000 --- a/.kokoro/test-samples-impl.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# `-e` enables the script to automatically fail when a command fails -# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero -set -eo pipefail -# Enables `**` to include files nested inside sub-folders -shopt -s globstar - -# Exit early if samples don't exist -if ! find samples -name 'requirements.txt' | grep -q .; then - echo "No tests run. './samples/**/requirements.txt' not found" - exit 0 -fi - -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 - -# Debug: show build environment -env | grep KOKORO - -# Install nox -python3.9 -m pip install --upgrade --quiet nox virtualenv - -# Use secrets acessor service account to get secrets -if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then - gcloud auth activate-service-account \ - --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ - --project="cloud-devrel-kokoro-resources" -fi - -# This script will create 3 files: -# - testing/test-env.sh -# - testing/service-account.json -# - testing/client-secrets.json -./scripts/decrypt-secrets.sh - -source ./testing/test-env.sh -export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json - -# For cloud-run session, we activate the service account for gcloud sdk. -gcloud auth activate-service-account \ - --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" - -export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json - -echo -e "\n******************** TESTING PROJECTS ********************" - -# Switch to 'fail at end' to allow all tests to complete before exiting. -set +e -# Use RTN to return a non-zero value if the test fails. -RTN=0 -ROOT=$(pwd) -# Find all requirements.txt in the samples directory (may break on whitespace). -for file in samples/**/requirements.txt; do - cd "$ROOT" - # Navigate to the project folder. - file=$(dirname "$file") - cd "$file" - - echo "------------------------------------------------------------" - echo "- testing $file" - echo "------------------------------------------------------------" - - # Use nox to execute the tests for the project. - python3.9 -m nox -s "$RUN_TESTS_SESSION" - EXIT=$? - - # If this is a periodic build, send the test log to the FlakyBot. - # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. - if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then - chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot - $KOKORO_GFILE_DIR/linux_amd64/flakybot - fi - - if [[ $EXIT -ne 0 ]]; then - RTN=1 - echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" - else - echo -e "\n Testing completed.\n" - fi - -done -cd "$ROOT" - -# Workaround for Kokoro permissions issue: delete secrets -rm testing/{test-env.sh,client-secrets.json,service-account.json} - -exit "$RTN" diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh deleted file mode 100755 index 7933d8201..000000000 --- a/.kokoro/test-samples.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The default test runner for samples. -# -# For periodic builds, we rewinds the repo to the latest release, and -# run test-samples-impl.sh. - -# `-e` enables the script to automatically fail when a command fails -# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero -set -eo pipefail -# Enables `**` to include files nested inside sub-folders -shopt -s globstar - -# Run periodic samples tests at latest release -if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then - # preserving the test runner implementation. - cp .kokoro/test-samples-impl.sh "${TMPDIR}/test-samples-impl.sh" - echo "--- IMPORTANT IMPORTANT IMPORTANT ---" - echo "Now we rewind the repo back to the latest release..." - LATEST_RELEASE=$(git describe --abbrev=0 --tags) - git checkout $LATEST_RELEASE - echo "The current head is: " - echo $(git rev-parse --verify HEAD) - echo "--- IMPORTANT IMPORTANT IMPORTANT ---" - # move back the test runner implementation if there's no file. - if [ ! -f .kokoro/test-samples-impl.sh ]; then - cp "${TMPDIR}/test-samples-impl.sh" .kokoro/test-samples-impl.sh - fi -fi - -exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh deleted file mode 100755 index 48f796997..000000000 --- a/.kokoro/trampoline.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -# Always run the cleanup script, regardless of the success of bouncing into -# the container. -function cleanup() { - chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh - ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh - echo "cleanup"; -} -trap cleanup EXIT - -$(dirname $0)/populate-secrets.sh # Secret Manager secrets. -python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" \ No newline at end of file diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh deleted file mode 100755 index 35fa52923..000000000 --- a/.kokoro/trampoline_v2.sh +++ /dev/null @@ -1,487 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# trampoline_v2.sh -# -# This script does 3 things. -# -# 1. Prepare the Docker image for the test -# 2. Run the Docker with appropriate flags to run the test -# 3. Upload the newly built Docker image -# -# in a way that is somewhat compatible with trampoline_v1. -# -# To run this script, first download few files from gcs to /dev/shm. -# (/dev/shm is passed into the container as KOKORO_GFILE_DIR). -# -# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm -# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm -# -# Then run the script. -# .kokoro/trampoline_v2.sh -# -# These environment variables are required: -# TRAMPOLINE_IMAGE: The docker image to use. -# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. -# -# You can optionally change these environment variables: -# TRAMPOLINE_IMAGE_UPLOAD: -# (true|false): Whether to upload the Docker image after the -# successful builds. -# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. -# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. -# Defaults to /workspace. -# Potentially there are some repo specific envvars in .trampolinerc in -# the project root. - - -set -euo pipefail - -TRAMPOLINE_VERSION="2.0.5" - -if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then - readonly IO_COLOR_RED="$(tput setaf 1)" - readonly IO_COLOR_GREEN="$(tput setaf 2)" - readonly IO_COLOR_YELLOW="$(tput setaf 3)" - readonly IO_COLOR_RESET="$(tput sgr0)" -else - readonly IO_COLOR_RED="" - readonly IO_COLOR_GREEN="" - readonly IO_COLOR_YELLOW="" - readonly IO_COLOR_RESET="" -fi - -function function_exists { - [ $(LC_ALL=C type -t $1)"" == "function" ] -} - -# Logs a message using the given color. The first argument must be one -# of the IO_COLOR_* variables defined above, such as -# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the -# given color. The log message will also have an RFC-3339 timestamp -# prepended (in UTC). You can disable the color output by setting -# TERM=vt100. -function log_impl() { - local color="$1" - shift - local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" - echo "================================================================" - echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" - echo "================================================================" -} - -# Logs the given message with normal coloring and a timestamp. -function log() { - log_impl "${IO_COLOR_RESET}" "$@" -} - -# Logs the given message in green with a timestamp. -function log_green() { - log_impl "${IO_COLOR_GREEN}" "$@" -} - -# Logs the given message in yellow with a timestamp. -function log_yellow() { - log_impl "${IO_COLOR_YELLOW}" "$@" -} - -# Logs the given message in red with a timestamp. -function log_red() { - log_impl "${IO_COLOR_RED}" "$@" -} - -readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) -readonly tmphome="${tmpdir}/h" -mkdir -p "${tmphome}" - -function cleanup() { - rm -rf "${tmpdir}" -} -trap cleanup EXIT - -RUNNING_IN_CI="${RUNNING_IN_CI:-false}" - -# The workspace in the container, defaults to /workspace. -TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" - -pass_down_envvars=( - # TRAMPOLINE_V2 variables. - # Tells scripts whether they are running as part of CI or not. - "RUNNING_IN_CI" - # Indicates which CI system we're in. - "TRAMPOLINE_CI" - # Indicates the version of the script. - "TRAMPOLINE_VERSION" -) - -log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" - -# Detect which CI systems we're in. If we're in any of the CI systems -# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be -# the name of the CI system. Both envvars will be passing down to the -# container for telling which CI system we're in. -if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then - # descriptive env var for indicating it's on CI. - RUNNING_IN_CI="true" - TRAMPOLINE_CI="kokoro" - if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then - if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then - log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." - exit 1 - fi - # This service account will be activated later. - TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" - else - if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then - gcloud auth list - fi - log_yellow "Configuring Container Registry access" - gcloud auth configure-docker --quiet - fi - pass_down_envvars+=( - # KOKORO dynamic variables. - "KOKORO_BUILD_NUMBER" - "KOKORO_BUILD_ID" - "KOKORO_JOB_NAME" - "KOKORO_GIT_COMMIT" - "KOKORO_GITHUB_COMMIT" - "KOKORO_GITHUB_PULL_REQUEST_NUMBER" - "KOKORO_GITHUB_PULL_REQUEST_COMMIT" - # For FlakyBot - "KOKORO_GITHUB_COMMIT_URL" - "KOKORO_GITHUB_PULL_REQUEST_URL" - ) -elif [[ "${TRAVIS:-}" == "true" ]]; then - RUNNING_IN_CI="true" - TRAMPOLINE_CI="travis" - pass_down_envvars+=( - "TRAVIS_BRANCH" - "TRAVIS_BUILD_ID" - "TRAVIS_BUILD_NUMBER" - "TRAVIS_BUILD_WEB_URL" - "TRAVIS_COMMIT" - "TRAVIS_COMMIT_MESSAGE" - "TRAVIS_COMMIT_RANGE" - "TRAVIS_JOB_NAME" - "TRAVIS_JOB_NUMBER" - "TRAVIS_JOB_WEB_URL" - "TRAVIS_PULL_REQUEST" - "TRAVIS_PULL_REQUEST_BRANCH" - "TRAVIS_PULL_REQUEST_SHA" - "TRAVIS_PULL_REQUEST_SLUG" - "TRAVIS_REPO_SLUG" - "TRAVIS_SECURE_ENV_VARS" - "TRAVIS_TAG" - ) -elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then - RUNNING_IN_CI="true" - TRAMPOLINE_CI="github-workflow" - pass_down_envvars+=( - "GITHUB_WORKFLOW" - "GITHUB_RUN_ID" - "GITHUB_RUN_NUMBER" - "GITHUB_ACTION" - "GITHUB_ACTIONS" - "GITHUB_ACTOR" - "GITHUB_REPOSITORY" - "GITHUB_EVENT_NAME" - "GITHUB_EVENT_PATH" - "GITHUB_SHA" - "GITHUB_REF" - "GITHUB_HEAD_REF" - "GITHUB_BASE_REF" - ) -elif [[ "${CIRCLECI:-}" == "true" ]]; then - RUNNING_IN_CI="true" - TRAMPOLINE_CI="circleci" - pass_down_envvars+=( - "CIRCLE_BRANCH" - "CIRCLE_BUILD_NUM" - "CIRCLE_BUILD_URL" - "CIRCLE_COMPARE_URL" - "CIRCLE_JOB" - "CIRCLE_NODE_INDEX" - "CIRCLE_NODE_TOTAL" - "CIRCLE_PREVIOUS_BUILD_NUM" - "CIRCLE_PROJECT_REPONAME" - "CIRCLE_PROJECT_USERNAME" - "CIRCLE_REPOSITORY_URL" - "CIRCLE_SHA1" - "CIRCLE_STAGE" - "CIRCLE_USERNAME" - "CIRCLE_WORKFLOW_ID" - "CIRCLE_WORKFLOW_JOB_ID" - "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" - "CIRCLE_WORKFLOW_WORKSPACE_ID" - ) -fi - -# Configure the service account for pulling the docker image. -function repo_root() { - local dir="$1" - while [[ ! -d "${dir}/.git" ]]; do - dir="$(dirname "$dir")" - done - echo "${dir}" -} - -# Detect the project root. In CI builds, we assume the script is in -# the git tree and traverse from there, otherwise, traverse from `pwd` -# to find `.git` directory. -if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then - PROGRAM_PATH="$(realpath "$0")" - PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" - PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" -else - PROJECT_ROOT="$(repo_root $(pwd))" -fi - -log_yellow "Changing to the project root: ${PROJECT_ROOT}." -cd "${PROJECT_ROOT}" - -# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need -# to use this environment variable in `PROJECT_ROOT`. -if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then - - mkdir -p "${tmpdir}/gcloud" - gcloud_config_dir="${tmpdir}/gcloud" - - log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." - export CLOUDSDK_CONFIG="${gcloud_config_dir}" - - log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." - gcloud auth activate-service-account \ - --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" - log_yellow "Configuring Container Registry access" - gcloud auth configure-docker --quiet -fi - -required_envvars=( - # The basic trampoline configurations. - "TRAMPOLINE_IMAGE" - "TRAMPOLINE_BUILD_FILE" -) - -if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then - source "${PROJECT_ROOT}/.trampolinerc" -fi - -log_yellow "Checking environment variables." -for e in "${required_envvars[@]}" -do - if [[ -z "${!e:-}" ]]; then - log "Missing ${e} env var. Aborting." - exit 1 - fi -done - -# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 -# script: e.g. "github/repo-name/.kokoro/run_tests.sh" -TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" -log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" - -# ignore error on docker operations and test execution -set +e - -log_yellow "Preparing Docker image." -# We only download the docker image in CI builds. -if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then - # Download the docker image specified by `TRAMPOLINE_IMAGE` - - # We may want to add --max-concurrent-downloads flag. - - log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." - if docker pull "${TRAMPOLINE_IMAGE}"; then - log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." - has_image="true" - else - log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." - has_image="false" - fi -else - # For local run, check if we have the image. - if docker images "${TRAMPOLINE_IMAGE}:latest" | grep "${TRAMPOLINE_IMAGE}"; then - has_image="true" - else - has_image="false" - fi -fi - - -# The default user for a Docker container has uid 0 (root). To avoid -# creating root-owned files in the build directory we tell docker to -# use the current user ID. -user_uid="$(id -u)" -user_gid="$(id -g)" -user_name="$(id -un)" - -# To allow docker in docker, we add the user to the docker group in -# the host os. -docker_gid=$(cut -d: -f3 < <(getent group docker)) - -update_cache="false" -if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then - # Build the Docker image from the source. - context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") - docker_build_flags=( - "-f" "${TRAMPOLINE_DOCKERFILE}" - "-t" "${TRAMPOLINE_IMAGE}" - "--build-arg" "UID=${user_uid}" - "--build-arg" "USERNAME=${user_name}" - ) - if [[ "${has_image}" == "true" ]]; then - docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") - fi - - log_yellow "Start building the docker image." - if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then - echo "docker build" "${docker_build_flags[@]}" "${context_dir}" - fi - - # ON CI systems, we want to suppress docker build logs, only - # output the logs when it fails. - if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then - if docker build "${docker_build_flags[@]}" "${context_dir}" \ - > "${tmpdir}/docker_build.log" 2>&1; then - if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then - cat "${tmpdir}/docker_build.log" - fi - - log_green "Finished building the docker image." - update_cache="true" - else - log_red "Failed to build the Docker image, aborting." - log_yellow "Dumping the build logs:" - cat "${tmpdir}/docker_build.log" - exit 1 - fi - else - if docker build "${docker_build_flags[@]}" "${context_dir}"; then - log_green "Finished building the docker image." - update_cache="true" - else - log_red "Failed to build the Docker image, aborting." - exit 1 - fi - fi -else - if [[ "${has_image}" != "true" ]]; then - log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." - exit 1 - fi -fi - -# We use an array for the flags so they are easier to document. -docker_flags=( - # Remove the container after it exists. - "--rm" - - # Use the host network. - "--network=host" - - # Run in priviledged mode. We are not using docker for sandboxing or - # isolation, just for packaging our dev tools. - "--privileged" - - # Run the docker script with the user id. Because the docker image gets to - # write in ${PWD} you typically want this to be your user id. - # To allow docker in docker, we need to use docker gid on the host. - "--user" "${user_uid}:${docker_gid}" - - # Pass down the USER. - "--env" "USER=${user_name}" - - # Mount the project directory inside the Docker container. - "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" - "--workdir" "${TRAMPOLINE_WORKSPACE}" - "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" - - # Mount the temporary home directory. - "--volume" "${tmphome}:/h" - "--env" "HOME=/h" - - # Allow docker in docker. - "--volume" "/var/run/docker.sock:/var/run/docker.sock" - - # Mount the /tmp so that docker in docker can mount the files - # there correctly. - "--volume" "/tmp:/tmp" - # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR - # TODO(tmatsuo): This part is not portable. - "--env" "TRAMPOLINE_SECRET_DIR=/secrets" - "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" - "--env" "KOKORO_GFILE_DIR=/secrets/gfile" - "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" - "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" -) - -# Add an option for nicer output if the build gets a tty. -if [[ -t 0 ]]; then - docker_flags+=("-it") -fi - -# Passing down env vars -for e in "${pass_down_envvars[@]}" -do - if [[ -n "${!e:-}" ]]; then - docker_flags+=("--env" "${e}=${!e}") - fi -done - -# If arguments are given, all arguments will become the commands run -# in the container, otherwise run TRAMPOLINE_BUILD_FILE. -if [[ $# -ge 1 ]]; then - log_yellow "Running the given commands '" "${@:1}" "' in the container." - readonly commands=("${@:1}") - if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then - echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" - fi - docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" -else - log_yellow "Running the tests in a Docker container." - docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") - if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then - echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" - fi - docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" -fi - - -test_retval=$? - -if [[ ${test_retval} -eq 0 ]]; then - log_green "Build finished with ${test_retval}" -else - log_red "Build finished with ${test_retval}" -fi - -# Only upload it when the test passes. -if [[ "${update_cache}" == "true" ]] && \ - [[ $test_retval == 0 ]] && \ - [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then - log_yellow "Uploading the Docker image." - if docker push "${TRAMPOLINE_IMAGE}"; then - log_green "Finished uploading the Docker image." - else - log_red "Failed uploading the Docker image." - fi - # Call trampoline_after_upload_hook if it's defined. - if function_exists trampoline_after_upload_hook; then - trampoline_after_upload_hook - fi - -fi - -exit "${test_retval}"