From c38a953766eff1faebb882260ab044c89d8ae3d6 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Mon, 30 Jun 2025 14:31:57 -0400 Subject: [PATCH 1/3] build: use slackware build number --- plugin/builder/build-plugin.ts | 9 +- plugin/builder/build-txz.ts | 2 +- plugin/builder/cli/common-environment.ts | 5 +- .../builder/cli/setup-plugin-environment.ts | 99 +++++++++++++------ plugin/builder/utils/bucket-urls.ts | 13 ++- plugin/builder/utils/consts.ts | 18 +++- plugin/builder/utils/paths.ts | 10 +- plugin/package.json | 2 +- 8 files changed, 109 insertions(+), 49 deletions(-) diff --git a/plugin/builder/build-plugin.ts b/plugin/builder/build-plugin.ts index 4c85155782..4376d94bbe 100644 --- a/plugin/builder/build-plugin.ts +++ b/plugin/builder/build-plugin.ts @@ -25,8 +25,8 @@ const checkGit = async () => { } }; -const moveTxzFile = async ({txzPath, apiVersion}: Pick) => { - const txzName = getTxzName(apiVersion); +const moveTxzFile = async ({txzPath, apiVersion, buildNumber}: Pick) => { + const txzName = getTxzName({version: apiVersion, build: buildNumber.toString()}); const targetPath = join(deployDir, txzName); // Ensure the txz always has the full version name @@ -54,6 +54,7 @@ function updateEntityValue( const buildPlugin = async ({ pluginVersion, baseUrl, + buildNumber, tag, txzSha256, releaseNotes, @@ -72,9 +73,9 @@ const buildPlugin = async ({ arch: defaultArch, build: defaultBuild, plugin_url: getPluginUrl({ baseUrl, tag }), - txz_url: getMainTxzUrl({ baseUrl, apiVersion, tag }), + txz_url: getMainTxzUrl({ baseUrl, apiVersion, tag, build: buildNumber.toString() }), txz_sha256: txzSha256, - txz_name: getTxzName(apiVersion), + txz_name: getTxzName({version: apiVersion, build: buildNumber.toString()}), ...(tag ? { tag } : {}), }; diff --git a/plugin/builder/build-txz.ts b/plugin/builder/build-txz.ts index 0a69df3dcc..524676e8af 100644 --- a/plugin/builder/build-txz.ts +++ b/plugin/builder/build-txz.ts @@ -158,7 +158,7 @@ const buildTxz = async (validatedEnv: TxzEnv) => { const version = validatedEnv.apiVersion; // Always use version when getting txz name - const txzName = getTxzName(version); + const txzName = getTxzName({ version, build: validatedEnv.buildNumber.toString() }); console.log(`Package name: ${txzName}`); const txzPath = join(validatedEnv.txzOutputDir, txzName); diff --git a/plugin/builder/cli/common-environment.ts b/plugin/builder/cli/common-environment.ts index 2a0d3890c3..261fce9125 100644 --- a/plugin/builder/cli/common-environment.ts +++ b/plugin/builder/cli/common-environment.ts @@ -10,6 +10,8 @@ export const baseEnvSchema = z.object({ apiVersion: z.string(), baseUrl: z.string().url(), tag: z.string().optional().default(""), + /** i.e. Slackware build number */ + buildNumber: z.coerce.number().int().default(1), }); export type BaseEnv = z.infer; @@ -43,5 +45,6 @@ export const addCommonOptions = (program: Command) => { "--tag ", "Tag (used for PR and staging builds)", process.env.TAG - ); + ) + .option("--build-number ", "Build number"); }; diff --git a/plugin/builder/cli/setup-plugin-environment.ts b/plugin/builder/cli/setup-plugin-environment.ts index 81065c74e8..7633140a39 100644 --- a/plugin/builder/cli/setup-plugin-environment.ts +++ b/plugin/builder/cli/setup-plugin-environment.ts @@ -8,22 +8,47 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; import { baseEnvSchema, addCommonOptions } from "./common-environment"; -const safeParseEnvSchema = baseEnvSchema.extend({ - txzPath: z.string().refine((val) => val.endsWith(".txz"), { - message: "TXZ Path must end with .txz", - }), +const basePluginSchema = baseEnvSchema.extend({ + txzPath: z + .string() + .refine((val) => val.endsWith(".txz"), { + message: "TXZ Path must end with .txz", + }) + .optional(), pluginVersion: z.string().regex(/^\d{4}\.\d{2}\.\d{2}\.\d{4}$/, { message: "Plugin version must be in the format YYYY.MM.DD.HHMM", }), releaseNotesPath: z.string().optional(), }); -const pluginEnvSchema = safeParseEnvSchema.extend({ - releaseNotes: z.string().nonempty("Release notes are required"), - txzSha256: z.string().refine((val) => val.length === 64, { - message: "TXZ SHA256 must be 64 characters long", - }), -}); +const safeParseEnvSchema = basePluginSchema.transform((data) => ({ + ...data, + txzPath: + data.txzPath || + getTxzPath({ + startingDir: process.cwd(), + version: data.apiVersion, + build: data.buildNumber.toString(), + }), +})); + +const pluginEnvSchema = basePluginSchema + .extend({ + releaseNotes: z.string().nonempty("Release notes are required"), + txzSha256: z.string().refine((val) => val.length === 64, { + message: "TXZ SHA256 must be 64 characters long", + }), + }) + .transform((data) => ({ + ...data, + txzPath: + data.txzPath || + getTxzPath({ + startingDir: process.cwd(), + version: data.apiVersion, + build: data.buildNumber.toString(), + }), + })); export type PluginEnv = z.infer; @@ -36,7 +61,11 @@ export type PluginEnv = z.infer; * @returns Object containing the resolved txz path and SHA256 hash * @throws Error if no valid txz file can be found */ -export const resolveTxzPath = async (txzPath: string, apiVersion: string, isCi?: boolean): Promise<{path: string, sha256: string}> => { +export const resolveTxzPath = async ( + txzPath: string, + apiVersion: string, + isCi?: boolean +): Promise<{ path: string; sha256: string }> => { if (existsSync(txzPath)) { await access(txzPath, constants.F_OK); console.log("Reading txz file from:", txzPath); @@ -46,35 +75,37 @@ export const resolveTxzPath = async (txzPath: string, apiVersion: string, isCi?: } return { path: txzPath, - sha256: getSha256(txzFile) + sha256: getSha256(txzFile), }; } console.log(`TXZ path not found at: ${txzPath}`); console.log(`Attempting to find TXZ using apiVersion: ${apiVersion}`); - + // Try different formats of generated TXZ name const deployDir = join(process.cwd(), "deploy"); - + // Try with exact apiVersion format const alternativePaths = [ join(deployDir, `dynamix.unraid.net-${apiVersion}-x86_64-1.txz`), ]; - + // In CI, we sometimes see unusual filenames, so try a glob-like approach if (isCi) { console.log("Checking for possible TXZ files in deploy directory"); - + try { // Using node's filesystem APIs to scan the directory - const fs = require('fs'); + const fs = require("fs"); const deployFiles = fs.readdirSync(deployDir); - + // Find any txz file that contains the apiVersion for (const file of deployFiles) { - if (file.endsWith('.txz') && - file.includes('dynamix.unraid.net') && - file.includes(apiVersion.split('+')[0])) { + if ( + file.endsWith(".txz") && + file.includes("dynamix.unraid.net") && + file.includes(apiVersion.split("+")[0]) + ) { alternativePaths.push(join(deployDir, file)); } } @@ -82,7 +113,7 @@ export const resolveTxzPath = async (txzPath: string, apiVersion: string, isCi?: console.log(`Error scanning deploy directory: ${error}`); } } - + // Check each path for (const path of alternativePaths) { if (existsSync(path)) { @@ -96,14 +127,16 @@ export const resolveTxzPath = async (txzPath: string, apiVersion: string, isCi?: } return { path, - sha256: getSha256(txzFile) + sha256: getSha256(txzFile), }; } console.log(`Could not find TXZ at: ${path}`); } - + // If we get here, we couldn't find a valid txz file - throw new Error(`Could not find any valid TXZ file. Tried original path: ${txzPath} and alternatives.`); + throw new Error( + `Could not find any valid TXZ file. Tried original path: ${txzPath} and alternatives.` + ); }; export const validatePluginEnv = async ( @@ -127,7 +160,11 @@ export const validatePluginEnv = async ( } // Resolve and validate the txz path - const { path, sha256 } = await resolveTxzPath(safeEnv.txzPath, safeEnv.apiVersion, safeEnv.ci); + const { path, sha256 } = await resolveTxzPath( + safeEnv.txzPath, + safeEnv.apiVersion, + safeEnv.ci + ); envArgs.txzPath = path; envArgs.txzSha256 = sha256; @@ -142,8 +179,9 @@ export const validatePluginEnv = async ( export const getPluginVersion = () => { const now = new Date(); - - const formatUtcComponent = (component: number) => String(component).padStart(2, '0'); + + const formatUtcComponent = (component: number) => + String(component).padStart(2, "0"); const year = now.getUTCFullYear(); const month = formatUtcComponent(now.getUTCMonth() + 1); @@ -162,13 +200,12 @@ export const setupPluginEnv = async (argv: string[]): Promise => { // Add common options addCommonOptions(program); - + // Add plugin-specific options program .option( "--txz-path ", - "Path to built package, will be used to generate the SHA256 and renamed with the plugin version", - getTxzPath({ startingDir: process.cwd(), pluginVersion: process.env.API_VERSION }) + "Path to built package, will be used to generate the SHA256 and renamed with the plugin version" ) .option( "--plugin-version ", diff --git a/plugin/builder/utils/bucket-urls.ts b/plugin/builder/utils/bucket-urls.ts index fc0f31c7a0..da25bad11f 100644 --- a/plugin/builder/utils/bucket-urls.ts +++ b/plugin/builder/utils/bucket-urls.ts @@ -1,4 +1,11 @@ -import { getTxzName, LOCAL_BUILD_TAG, pluginNameWithExt, defaultArch, defaultBuild } from "./consts"; +import { + getTxzName, + LOCAL_BUILD_TAG, + pluginNameWithExt, + defaultArch, + defaultBuild, + TxzNameParams, +} from "./consts"; // Define a common interface for URL parameters interface UrlParams { @@ -6,7 +13,7 @@ interface UrlParams { tag?: string; } -interface TxzUrlParams extends UrlParams { +interface TxzUrlParams extends UrlParams, TxzNameParams { apiVersion: string; } @@ -47,4 +54,4 @@ export const getPluginUrl = (params: UrlParams): string => * ex. returns = BASE_URL/TAG/dynamix.unraid.net-4.1.3-x86_64-1.txz */ export const getMainTxzUrl = (params: TxzUrlParams): string => - getAssetUrl(params, getTxzName(params.apiVersion, defaultArch, defaultBuild)); + getAssetUrl(params, getTxzName(params)); diff --git a/plugin/builder/utils/consts.ts b/plugin/builder/utils/consts.ts index 62a5349948..883022e311 100644 --- a/plugin/builder/utils/consts.ts +++ b/plugin/builder/utils/consts.ts @@ -5,9 +5,21 @@ export const pluginNameWithExt = `${pluginName}.plg` as const; export const defaultArch = "x86_64" as const; export const defaultBuild = "1" as const; +export interface TxzNameParams { + version?: string; + arch?: string; + build?: string; +} + // Get the txz name following Slackware naming convention: name-version-arch-build.txz -export const getTxzName = (version?: string, arch: string = defaultArch, build: string = defaultBuild) => - version ? `${pluginName}-${version}-${arch}-${build}.txz` : `${pluginName}.txz`; +export const getTxzName = ({ + version, + arch = defaultArch, + build = defaultBuild, +}: TxzNameParams) => + version + ? `${pluginName}-${version}-${arch}-${build}.txz` + : `${pluginName}.txz`; export const startingDir = process.cwd(); export const BASE_URLS = { @@ -15,4 +27,4 @@ export const BASE_URLS = { PREVIEW: "https://preview.dl.unraid.net/unraid-api", } as const; -export const LOCAL_BUILD_TAG = "LOCAL_PLUGIN_BUILD" as const; \ No newline at end of file +export const LOCAL_BUILD_TAG = "LOCAL_PLUGIN_BUILD" as const; diff --git a/plugin/builder/utils/paths.ts b/plugin/builder/utils/paths.ts index 146c1889ae..20a8cdf364 100644 --- a/plugin/builder/utils/paths.ts +++ b/plugin/builder/utils/paths.ts @@ -4,15 +4,14 @@ import { pluginName, pluginNameWithExt, startingDir, + TxzNameParams, } from "./consts"; export interface PathConfig { startingDir: string; } -export interface TxzPathConfig extends PathConfig { - pluginVersion?: string; -} +export interface TxzPathConfig extends PathConfig, TxzNameParams {} export const deployDir = "deploy" as const; @@ -53,7 +52,8 @@ export function getDeployPluginPath({ startingDir }: PathConfig): string { */ export function getTxzPath({ startingDir, - pluginVersion, + version, + build, }: TxzPathConfig): string { - return join(startingDir, deployDir, getTxzName(pluginVersion)); + return join(startingDir, deployDir, getTxzName({ version, build })); } diff --git a/plugin/package.json b/plugin/package.json index 78e7eef70e..6ad53dca59 100644 --- a/plugin/package.json +++ b/plugin/package.json @@ -12,7 +12,7 @@ "tsx": "^4.19.2", "zod": "^3.24.1", "zx": "^8.3.2" -}, + }, "type": "module", "license": "GPL-2.0-or-later", "scripts": { From debafbef82d41e9e9346040863e23bc14d1d7b5f Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Mon, 30 Jun 2025 14:44:59 -0400 Subject: [PATCH 2/3] update ci workflows --- .github/workflows/build-plugin.yml | 8 ++++++-- .github/workflows/main.yml | 12 ++++++++++++ pnpm-lock.yaml | 7 +++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-plugin.yml b/.github/workflows/build-plugin.yml index 5d3fd0b0e9..a52acb9f1d 100644 --- a/.github/workflows/build-plugin.yml +++ b/.github/workflows/build-plugin.yml @@ -23,6 +23,10 @@ on: type: string required: true description: "Base URL for the plugin builds" + BUILD_NUMBER: + type: string + required: true + description: "Build number for the plugin builds" secrets: CF_ACCESS_KEY_ID: required: true @@ -108,8 +112,8 @@ jobs: id: build-plugin run: | cd ${{ github.workspace }}/plugin - pnpm run build:txz --tag="${{ inputs.TAG }}" --base-url="${{ inputs.BASE_URL }}" --api-version="${{ steps.vars.outputs.API_VERSION }}" - pnpm run build:plugin --tag="${{ inputs.TAG }}" --base-url="${{ inputs.BASE_URL }}" --api-version="${{ steps.vars.outputs.API_VERSION }}" + pnpm run build:txz --tag="${{ inputs.TAG }}" --base-url="${{ inputs.BASE_URL }}" --api-version="${{ steps.vars.outputs.API_VERSION }}" --build-number="${{ inputs.BUILD_NUMBER }}" + pnpm run build:plugin --tag="${{ inputs.TAG }}" --base-url="${{ inputs.BASE_URL }}" --api-version="${{ steps.vars.outputs.API_VERSION }}" --build-number="${{ inputs.BUILD_NUMBER }}" - name: Ensure Plugin Files Exist run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 95caa90415..9f4e37d595 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -156,6 +156,8 @@ jobs: build-api: name: Build API runs-on: ubuntu-latest + outputs: + build_number: ${{ steps.buildnumber.outputs.build_number }} defaults: run: working-directory: api @@ -210,6 +212,14 @@ jobs: API_VERSION=$([[ -n "$IS_TAGGED" ]] && echo "$PACKAGE_LOCK_VERSION" || echo "${PACKAGE_LOCK_VERSION}+${GIT_SHA}") export API_VERSION echo "API_VERSION=${API_VERSION}" >> $GITHUB_ENV + echo "PACKAGE_LOCK_VERSION=${PACKAGE_LOCK_VERSION}" >> $GITHUB_OUTPUT + + - name: Generate build number + id: buildnumber + uses: onyxmueller/build-tag-number@v1 + with: + token: ${{secrets.github_token}} + prefix: ${{steps.vars.outputs.PACKAGE_LOCK_VERSION}} - name: Build run: | @@ -365,6 +375,7 @@ jobs: TAG: ${{ github.event.pull_request.number && format('PR{0}', github.event.pull_request.number) || '' }} BUCKET_PATH: ${{ github.event.pull_request.number && format('unraid-api/tag/PR{0}', github.event.pull_request.number) || 'unraid-api' }} BASE_URL: "https://preview.dl.unraid.net/unraid-api" + BUILD_NUMBER: ${{ needs.build-api.outputs.build_number }} secrets: CF_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }} CF_SECRET_ACCESS_KEY: ${{ secrets.CF_SECRET_ACCESS_KEY }} @@ -387,6 +398,7 @@ jobs: TAG: "" BUCKET_PATH: unraid-api BASE_URL: "https://stable.dl.unraid.net/unraid-api" + BUILD_NUMBER: ${{ needs.build-api.outputs.build_number }} secrets: CF_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }} CF_SECRET_ACCESS_KEY: ${{ secrets.CF_SECRET_ACCESS_KEY }} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 056b09d011..9767d9356e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11107,6 +11107,7 @@ packages: engines: {node: '>=0.6.0', teleport: '>=0.2.0'} deprecated: |- You or someone you depend on is using Q, the JavaScript Promise library that gave JavaScript developers strong feelings about promises. They can almost certainly migrate to the native JavaScript promise now. Thank you literally everyone for joining me in this bet against the odds. Be excellent to each other. + (For a CapTP with native promises, see @endo/eventual-send and @endo/captp) qs@6.13.0: @@ -21480,6 +21481,8 @@ snapshots: dependencies: tabbable: 6.2.0 + follow-redirects@1.15.9: {} + follow-redirects@1.15.9(debug@4.3.7): optionalDependencies: debug: 4.3.7 @@ -22152,7 +22155,7 @@ snapshots: http-proxy@1.18.1: dependencies: eventemitter3: 4.0.7 - follow-redirects: 1.15.9(debug@4.3.7) + follow-redirects: 1.15.9 requires-port: 1.0.0 transitivePeerDependencies: - debug @@ -26301,7 +26304,7 @@ snapshots: terser@5.43.1: dependencies: '@jridgewell/source-map': 0.3.6 - acorn: 8.14.1 + acorn: 8.15.0 commander: 2.20.3 source-map-support: 0.5.21 From ef15fd2ad83eab2feb632e8452f5af8d6d5b5cbe Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Mon, 30 Jun 2025 15:10:54 -0400 Subject: [PATCH 3/3] fix version piping to txz url --- plugin/builder/build-plugin.ts | 37 ++++++++++++++++++++++------- plugin/builder/utils/bucket-urls.ts | 4 +--- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/plugin/builder/build-plugin.ts b/plugin/builder/build-plugin.ts index 4376d94bbe..7238b8dfa9 100644 --- a/plugin/builder/build-plugin.ts +++ b/plugin/builder/build-plugin.ts @@ -2,7 +2,13 @@ import { readFile, writeFile, mkdir, rename } from "fs/promises"; import { $ } from "zx"; import { escape as escapeHtml } from "html-sloppy-escaper"; import { dirname, join } from "node:path"; -import { getTxzName, pluginName, startingDir, defaultArch, defaultBuild } from "./utils/consts"; +import { + getTxzName, + pluginName, + startingDir, + defaultArch, + defaultBuild, +} from "./utils/consts"; import { getPluginUrl } from "./utils/bucket-urls"; import { getMainTxzUrl } from "./utils/bucket-urls"; import { @@ -25,10 +31,17 @@ const checkGit = async () => { } }; -const moveTxzFile = async ({txzPath, apiVersion, buildNumber}: Pick) => { - const txzName = getTxzName({version: apiVersion, build: buildNumber.toString()}); +const moveTxzFile = async ({ + txzPath, + apiVersion, + buildNumber, +}: Pick) => { + const txzName = getTxzName({ + version: apiVersion, + build: buildNumber.toString(), + }); const targetPath = join(deployDir, txzName); - + // Ensure the txz always has the full version name if (txzPath !== targetPath) { console.log(`Ensuring TXZ has correct name: ${txzPath} -> ${targetPath}`); @@ -61,7 +74,7 @@ const buildPlugin = async ({ apiVersion, }: PluginEnv) => { console.log(`API version: ${apiVersion}`); - + // Update plg file let plgContent = await readFile(getRootPluginPath({ startingDir }), "utf8"); @@ -71,11 +84,19 @@ const buildPlugin = async ({ version: pluginVersion, api_version: apiVersion, arch: defaultArch, - build: defaultBuild, + build: buildNumber.toString(), plugin_url: getPluginUrl({ baseUrl, tag }), - txz_url: getMainTxzUrl({ baseUrl, apiVersion, tag, build: buildNumber.toString() }), + txz_url: getMainTxzUrl({ + baseUrl, + tag, + version: apiVersion, + build: buildNumber.toString(), + }), txz_sha256: txzSha256, - txz_name: getTxzName({version: apiVersion, build: buildNumber.toString()}), + txz_name: getTxzName({ + version: apiVersion, + build: buildNumber.toString(), + }), ...(tag ? { tag } : {}), }; diff --git a/plugin/builder/utils/bucket-urls.ts b/plugin/builder/utils/bucket-urls.ts index da25bad11f..c3dd13a4c5 100644 --- a/plugin/builder/utils/bucket-urls.ts +++ b/plugin/builder/utils/bucket-urls.ts @@ -13,9 +13,7 @@ interface UrlParams { tag?: string; } -interface TxzUrlParams extends UrlParams, TxzNameParams { - apiVersion: string; -} +interface TxzUrlParams extends UrlParams, TxzNameParams {} /** * Get the bucket path for the given tag