Skip to content

Commit 47f15bc

Browse files
JamesMBartlettJamesMBartlett
committed
[artifact manifest] Generate manifest updates from mirror uploads.
Signed-off-by: James Bartlett <jamesbartlett@pixielabs.ai>
1 parent 90de775 commit 47f15bc

File tree

9 files changed

+260
-24
lines changed

9 files changed

+260
-24
lines changed

‎.github/workflows/cli_release.yaml‎

Lines changed: 74 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ jobs:
1919
needs: get-dev-image
2020
container:
2121
image: ${{ needs.get-dev-image.outputs.image-with-tag }}
22+
env:
23+
ARTIFACT_UPLOAD_LOG: "artifact_uploads.json"
2224
steps:
2325
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
2426
with:
@@ -65,11 +67,17 @@ jobs:
6567
with:
6668
name: linux-artifacts
6769
path: artifacts/
68-
- name: Update Manifest
70+
- name: Update GCS Manifest
6971
env:
7072
ARTIFACT_MANIFEST_BUCKET: "pixie-dev-public"
73+
# Use the old style versions file instead of the new updates for the gcs manifest.
74+
MANIFEST_UPDATES: ""
7175
GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
7276
run: ./ci/update_artifact_manifest.sh
77+
- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
78+
with:
79+
name: artifact-upload-log
80+
path: ${{ env.ARTIFACT_UPLOAD_LOG }}
7381
sign-release:
7482
name: Sign Release for MacOS
7583
runs-on: macos-latest
@@ -107,13 +115,18 @@ jobs:
107115
needs: [get-dev-image, sign-release]
108116
container:
109117
image: ${{ needs.get-dev-image.outputs.image-with-tag }}
118+
env:
119+
MANIFEST_UPDATES: "manifest_updates.json"
110120
steps:
111121
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
112122
with:
113123
fetch-depth: 0
114124
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
115125
with:
116126
name: macos-artifacts
127+
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
128+
with:
129+
name: artifact-upload-log
117130
- name: Import GPG key
118131
env:
119132
BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }}
@@ -128,6 +141,7 @@ jobs:
128141
REF: ${{ github.event.ref }}
129142
BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }}
130143
GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
144+
ARTIFACT_UPLOAD_LOG: "artifact_uploads.json"
131145
shell: bash
132146
run: |
133147
export TAG_NAME="${REF#*/tags/}"
@@ -138,8 +152,11 @@ jobs:
138152
with:
139153
name: macos-artifacts
140154
path: artifacts/
155+
- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
156+
with:
157+
name: manifest-updates
158+
path: ${{ env.MANIFEST_UPDATES }}
141159
create-github-release:
142-
if: ${{ !contains(github.event.ref, '-') }}
143160
name: Create Release on Github
144161
runs-on: ubuntu-latest
145162
needs: push-signed-artifacts
@@ -154,16 +171,68 @@ jobs:
154171
env:
155172
REF: ${{ github.event.ref }}
156173
GH_TOKEN: ${{ secrets.BUILDBOT_GH_API_TOKEN }}
157-
OWNER: pixie-io
158-
REPO: pixie
159174
shell: bash
160175
# yamllint disable rule:indentation
161176
run: |
162177
export TAG_NAME="${REF#*/tags/}"
163178
# actions/checkout doesn't get the tag annotation properly.
164179
git fetch origin tag "${TAG_NAME}" -f
165180
export changelog="$(git tag -l --format='%(contents)' "${TAG_NAME}")"
166-
gh release create "${TAG_NAME}" --title "CLI ${TAG_NAME#release/cli/}" \
181+
prerelease=""
182+
if [[ "${REF}" == *"-"* ]]; then
183+
prerelease="--prerelease"
184+
fi
185+
gh release create "${TAG_NAME}" "${prerelease}" \
186+
--title "CLI ${TAG_NAME#release/cli/}" \
167187
--notes $'Pixie CLI Release:\n'"${changelog}"
168188
gh release upload "${TAG_NAME}" linux-artifacts/* macos-artifacts/*
169189
# yamllint enable rule:indentation
190+
update-gh-artifacts-manifest:
191+
runs-on: ubuntu-latest-8-cores
192+
needs: [get-dev-image, create-github-release]
193+
container:
194+
image: ${{ needs.get-dev-image.outputs.image-with-tag }}
195+
concurrency: gh-pages
196+
permissions:
197+
contents: write
198+
steps:
199+
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
200+
with:
201+
fetch-depth: 0
202+
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
203+
with:
204+
ref: gh-pages
205+
path: gh-pages
206+
- name: Add pwd to git safe dir
207+
run: |
208+
git config --global --add safe.directory `pwd`
209+
git config --global --add safe.directory "$(pwd)/gh-pages"
210+
- name: Import GPG key
211+
env:
212+
BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }}
213+
run: |
214+
echo "${BUILDBOT_GPG_KEY_B64}" | base64 --decode | gpg --no-tty --batch --import
215+
- name: Setup git
216+
shell: bash
217+
env:
218+
BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }}
219+
run: |
220+
git config --global user.name 'pixie-io-buildbot'
221+
git config --global user.email 'build@pixielabs.ai'
222+
git config --global user.signingkey "${BUILDBOT_GPG_KEY_ID}"
223+
git config --global commit.gpgsign true
224+
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
225+
id: download-artifact
226+
with:
227+
name: manifest-updates
228+
- name: Update gh-pages Manifest
229+
env:
230+
ARTIFACT_MANIFEST_PATH: "gh-pages/artifacts/manifest.json"
231+
MANIFEST_UPDATES: "manifest_updates.json"
232+
run: |
233+
./ci/update_artifact_manifest.sh
234+
cd gh-pages
235+
export ARTIFACT_MANIFEST_PATH="${ARTIFACT_MANIFEST_PATH##gh-pages/}"
236+
git add "${ARTIFACT_MANIFEST_PATH}" "${ARTIFACT_MANIFEST_PATH}.sha256"
237+
git commit -s -m "Update artifact manifest"
238+
git push origin "gh-pages"

‎.github/workflows/operator_release.yaml‎

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ jobs:
1919
needs: get-dev-image
2020
container:
2121
image: ${{ needs.get-dev-image.outputs.image-with-tag }}
22+
env:
23+
MANIFEST_UPDATES: "manifest_updates.json"
24+
ARTIFACT_UPLOAD_LOG: "artifact_uploads.json"
2225
steps:
2326
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
2427
with:
@@ -58,11 +61,17 @@ jobs:
5861
mkdir -p "${ARTIFACTS_DIR}"
5962
./ci/save_version_info.sh
6063
./ci/operator_build_release.sh
61-
- name: Update Manifest
64+
- name: Update GCS Manifest
6265
env:
6366
ARTIFACT_MANIFEST_BUCKET: "pixie-dev-public"
67+
# Use the old style versions file instead of the new updates for the gcs manifest.
68+
MANIFEST_UPDATES: ""
6469
GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
6570
run: ./ci/update_artifact_manifest.sh
71+
- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
72+
with:
73+
name: manifest-updates
74+
path: ${{ env.MANIFEST_UPDATES }}
6675
- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
6776
with:
6877
name: operator-artifacts
@@ -72,7 +81,6 @@ jobs:
7281
name: index-artifacts
7382
path: index.yaml
7483
create-github-release:
75-
if: ${{ !contains(github.event.ref, '-') }}
7684
name: Create Release on Github
7785
runs-on: ubuntu-latest
7886
needs: build-release
@@ -87,16 +95,19 @@ jobs:
8795
env:
8896
REF: ${{ github.event.ref }}
8997
GH_TOKEN: ${{ secrets.BUILDBOT_GH_API_TOKEN }}
90-
OWNER: pixie-io
91-
REPO: pixie
9298
shell: bash
9399
# yamllint disable rule:indentation
94100
run: |
95101
export TAG_NAME="${REF#*/tags/}"
96102
# actions/checkout doesn't get the tag annotation properly.
97103
git fetch origin tag "${TAG_NAME}" -f
98104
export changelog="$(git tag -l --format='%(contents)' "${TAG_NAME}")"
99-
gh release create "${TAG_NAME}" --title "Operator ${TAG_NAME#release/operator/}" \
105+
prerelease=""
106+
if [[ "${REF}" == *"-"* ]]; then
107+
prerelease="--prerelease"
108+
fi
109+
gh release create "${TAG_NAME}" "${prerelease}" \
110+
--title "Operator ${TAG_NAME#release/operator/}" \
100111
--notes $'Pixie Operator Release:\n'"${changelog}"
101112
gh release upload "${TAG_NAME}" operator-artifacts/*
102113
create-helm-chart:
@@ -134,3 +145,52 @@ jobs:
134145
git commit -s -m "Release Helm chart ${VERSION}"
135146
git push origin "gh-pages"
136147
# yamllint enable rule:indentation
148+
update-gh-artifacts-manifest:
149+
runs-on: ubuntu-latest-8-cores
150+
needs: [get-dev-image, create-github-release]
151+
container:
152+
image: ${{ needs.get-dev-image.outputs.image-with-tag }}
153+
concurrency: gh-pages
154+
permissions:
155+
contents: write
156+
steps:
157+
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
158+
with:
159+
fetch-depth: 0
160+
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
161+
with:
162+
ref: gh-pages
163+
path: gh-pages
164+
- name: Add pwd to git safe dir
165+
run: |
166+
git config --global --add safe.directory `pwd`
167+
git config --global --add safe.directory "$(pwd)/gh-pages"
168+
- name: Import GPG key
169+
env:
170+
BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }}
171+
run: |
172+
echo "${BUILDBOT_GPG_KEY_B64}" | base64 --decode | gpg --no-tty --batch --import
173+
- name: Setup git
174+
shell: bash
175+
env:
176+
BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }}
177+
run: |
178+
git config --global user.name 'pixie-io-buildbot'
179+
git config --global user.email 'build@pixielabs.ai'
180+
git config --global user.signingkey "${BUILDBOT_GPG_KEY_ID}"
181+
git config --global commit.gpgsign true
182+
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
183+
id: download-artifact
184+
with:
185+
name: manifest-updates
186+
- name: Update gh-pages Manifest
187+
env:
188+
ARTIFACT_MANIFEST_PATH: "gh-pages/artifacts/manifest.json"
189+
MANIFEST_UPDATES: "manifest_updates.json"
190+
run: |
191+
./ci/update_artifact_manifest.sh
192+
cd gh-pages
193+
export ARTIFACT_MANIFEST_PATH="${ARTIFACT_MANIFEST_PATH##gh-pages/}"
194+
git add "${ARTIFACT_MANIFEST_PATH}" "${ARTIFACT_MANIFEST_PATH}.sha256"
195+
git commit -s -m "Update artifact manifest"
196+
git push origin "gh-pages"

‎.github/workflows/vizier_release.yaml‎

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ jobs:
1919
needs: get-dev-image
2020
container:
2121
image: ${{ needs.get-dev-image.outputs.image-with-tag }}
22+
env:
23+
MANIFEST_UPDATES: "manifest_updates.json"
24+
ARTIFACT_UPLOAD_LOG: "artifact_uploads.json"
2225
steps:
2326
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
2427
with:
@@ -65,11 +68,17 @@ jobs:
6568
docs="$(mktemp)"
6669
bazel run //src/carnot/docstring:docstring -- --output_json "${docs}"
6770
gsutil cp "${docs}" "${PXL_DOCS_GCS_PATH}"
68-
- name: Update Manifest
71+
- name: Update GCS Manifest
6972
env:
7073
ARTIFACT_MANIFEST_BUCKET: "pixie-dev-public"
74+
# Use the old style versions file instead of the new updates for the gcs manifest.
75+
MANIFEST_UPDATES: ""
7176
GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
7277
run: ./ci/update_artifact_manifest.sh
78+
- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
79+
with:
80+
name: manifest-updates
81+
path: ${{ env.MANIFEST_UPDATES }}
7382
- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
7483
with:
7584
name: vizier-artifacts
@@ -79,7 +88,6 @@ jobs:
7988
name: index-artifacts
8089
path: index.yaml
8190
create-github-release:
82-
if: ${{ !contains(github.event.ref, '-') }}
8391
name: Create Release on Github
8492
runs-on: ubuntu-latest
8593
needs: build-release
@@ -94,16 +102,19 @@ jobs:
94102
env:
95103
REF: ${{ github.event.ref }}
96104
GH_TOKEN: ${{ secrets.BUILDBOT_GH_API_TOKEN }}
97-
OWNER: pixie-io
98-
REPO: pixie
99105
shell: bash
100106
# yamllint disable rule:indentation
101107
run: |
102108
export TAG_NAME="${REF#*/tags/}"
103109
# actions/checkout doesn't get the tag annotation properly.
104110
git fetch origin tag "${TAG_NAME}" -f
105111
export changelog="$(git tag -l --format='%(contents)' "${TAG_NAME}")"
106-
gh release create "${TAG_NAME}" --title "Vizier ${TAG_NAME#release/vizier/}" \
112+
prerelease=""
113+
if [[ "${REF}" == *"-"* ]]; then
114+
prerelease="--prerelease"
115+
fi
116+
gh release create "${TAG_NAME}" "${prerelease}" \
117+
--title "Vizier ${TAG_NAME#release/vizier/}" \
107118
--notes $'Pixie Vizier Release:\n'"${changelog}"
108119
gh release upload "${TAG_NAME}" vizier-artifacts/*
109120
# yamllint enable rule:indentation
@@ -142,3 +153,52 @@ jobs:
142153
git commit -s -m "Release Helm chart Vizier ${VERSION}"
143154
git push origin "gh-pages"
144155
# yamllint enable rule:indentation
156+
update-gh-artifacts-manifest:
157+
runs-on: ubuntu-latest-8-cores
158+
needs: [get-dev-image, create-github-release]
159+
container:
160+
image: ${{ needs.get-dev-image.outputs.image-with-tag }}
161+
concurrency: gh-pages
162+
permissions:
163+
contents: write
164+
steps:
165+
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
166+
with:
167+
fetch-depth: 0
168+
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
169+
with:
170+
ref: gh-pages
171+
path: gh-pages
172+
- name: Add pwd to git safe dir
173+
run: |
174+
git config --global --add safe.directory `pwd`
175+
git config --global --add safe.directory "$(pwd)/gh-pages"
176+
- name: Import GPG key
177+
env:
178+
BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }}
179+
run: |
180+
echo "${BUILDBOT_GPG_KEY_B64}" | base64 --decode | gpg --no-tty --batch --import
181+
- name: Setup git
182+
shell: bash
183+
env:
184+
BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }}
185+
run: |
186+
git config --global user.name 'pixie-io-buildbot'
187+
git config --global user.email 'build@pixielabs.ai'
188+
git config --global user.signingkey "${BUILDBOT_GPG_KEY_ID}"
189+
git config --global commit.gpgsign true
190+
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
191+
id: download-artifact
192+
with:
193+
name: manifest-updates
194+
- name: Update gh-pages Manifest
195+
env:
196+
ARTIFACT_MANIFEST_PATH: "gh-pages/artifacts/manifest.json"
197+
MANIFEST_UPDATES: "manifest_updates.json"
198+
run: |
199+
./ci/update_artifact_manifest.sh
200+
cd gh-pages
201+
export ARTIFACT_MANIFEST_PATH="${ARTIFACT_MANIFEST_PATH##gh-pages/}"
202+
git add "${ARTIFACT_MANIFEST_PATH}" "${ARTIFACT_MANIFEST_PATH}.sha256"
203+
git commit -s -m "Update artifact manifest"
204+
git push origin "gh-pages"

‎ci/artifact_mirrors.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# So the first mirror in the list will be pulled from first.
44
- name: gh-releases
55
type: gh-releases
6-
url_format: 'https://github.com/pixie-io/pixie/releases/download/release/${component}/v${version}/${artifact_name}'
6+
url_format: 'https://github.com/${gh_repo}/releases/download/release/${component}/v${version}/${artifact_name}'
77
- name: pixie-oss-gcs
88
type: gcs
99
bucket: pixie-dev-public

0 commit comments

Comments
 (0)