Merge pull request #12874 from CesiumGS/gallery-build-path-fix

Sandcastle gallery build path fix

Author: jjspace

packages/sandcastle/sandcastle.config.js

@@ -3,7 +3,7 @@ const config = {
   sourceUrl: "https://github.com/CesiumGS/cesium/blob/main/packages/sandcastle",
   publicDir: "./public",
   gallery: {
-    files: ["gallery/**/*"],
+    files: ["gallery"],
     searchOptions: {
       excerptLength: 10,
       ranking: {

packages/sandcastle/scripts/buildGallery.js

@@ -15,7 +15,7 @@ import createGalleryRecord from "./createGalleryRecord.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const defaultRootDirectory = join(__dirname, "..");
 const defaultPublicDirectory = "./public";
-const defaultGalleryFiles = ["gallery/**/*"];
+const defaultGalleryFiles = ["gallery"];
 const defaultThumbnailPath = "images/placeholder-thumbnail.jpg";
 const requiredMetadataKeys = ["title", "description"];
 const galleryItemConfig = /sandcastle\.(yml|yaml)/;
@@ -118,7 +118,7 @@ export async function buildGalleryList(options = {}) {
   };
 
   const galleryFiles = await globby(
-    galleryFilesPattern.map((pattern) => join(rootDirectory, pattern)),
+    galleryFilesPattern.map((pattern) => join(rootDirectory, pattern, "**/*")),
   );
   const yamlFiles = galleryFiles.filter((path) =>
     basename(path).match(galleryItemConfig),
@@ -261,7 +261,7 @@ export async function buildGalleryList(options = {}) {
   // regardless of if titles match the directory names
   output.entries.sort((a, b) => a.title.localeCompare(b.title));
 
-  const outputDirectory = join(publicDirectory, "gallery");
+  const outputDirectory = join(rootDirectory, publicDirectory, "gallery");
   await rimraf(outputDirectory);
   await mkdir(outputDirectory, { recursive: true });
 

scripts/build.js

@@ -1,10 +1,10 @@
-import child_process from "child_process";
-import { existsSync, readFileSync, statSync } from "fs";
-import { readFile, writeFile } from "fs/promises";
-import { EOL } from "os";
-import path from "path";
-import { createRequire } from "module";
-import { finished } from "stream/promises";
+import child_process from "node:child_process";
+import { existsSync, statSync } from "node:fs";
+import { readFile, writeFile } from "node:fs/promises";
+import { EOL } from "node:os";
+import path from "node:path";
+import { finished } from "node:stream/promises";
+import { fileURLToPath } from "node:url";
 
 import esbuild from "esbuild";
 import { globby } from "globby";
@@ -18,21 +18,23 @@ import { mkdirp } from "mkdirp";
 // This should match the scope of the dependencies of the root level package.json.
 const scope = "cesium";
 
-const require = createRequire(import.meta.url);
-const packageJson = require("../package.json");
-let version = packageJson.version;
-if (/\.0$/.test(version)) {
-  version = version.substring(0, version.length - 2);
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const projectRoot = path.join(__dirname, "..");
+const packageJsonPath = path.join(projectRoot, "package.json");
+
+async function getVersion() {
+  const data = await readFile(packageJsonPath, "utf8");
+  const { version } = JSON.parse(data);
+  return version;
 }
 
-const copyrightHeaderTemplate = readFileSync(
-  path.join("Source", "copyrightHeader.js"),
-  "utf8",
-);
-const combinedCopyrightHeader = copyrightHeaderTemplate.replace(
-  "${version}",
-  version,
-);
+async function getCopyrightHeader() {
+  const copyrightHeaderTemplate = await readFile(
+    path.join("Source", "copyrightHeader.js"),
+    "utf8",
+  );
+  return copyrightHeaderTemplate.replace("${version}", await getVersion());
+}
 
 function escapeCharacters(token) {
   return token.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
@@ -164,7 +166,7 @@ export async function bundleCesiumJs(options) {
   buildConfig.plugins = options.removePragmas ? [stripPragmaPlugin] : undefined;
   buildConfig.write = options.write;
   buildConfig.banner = {
-    js: combinedCopyrightHeader,
+    js: await getCopyrightHeader(),
   };
   // print errors immediately, and collect warnings so we can filter out known ones
   buildConfig.logLevel = "info";
@@ -288,6 +290,7 @@ function generateDeclaration(workspace, file) {
  * @returns {Buffer} contents
  */
 export async function createCesiumJs() {
+  const version = await getVersion();
   let contents = `export const VERSION = '${version}';\n`;
 
   // Iterate over each workspace and generate declarations for each file.
@@ -314,6 +317,7 @@ const workspaceSpecFiles = {
  * @returns {Buffer} contents
  */
 export async function createCombinedSpecList() {
+  const version = await getVersion();
   let contents = `export const VERSION = '${version}';\n`;
 
   for (const workspace of Object.keys(workspaceSpecFiles)) {
@@ -387,7 +391,7 @@ export async function bundleWorkers(options) {
     workerConfig.format = "esm";
     workerConfig.splitting = true;
     workerConfig.banner = {
-      js: combinedCopyrightHeader,
+      js: await getCopyrightHeader(),
     };
     workerConfig.entryPoints = workers;
     workerConfig.outdir = path.join(options.path, "Workers");
@@ -606,17 +610,31 @@ const externalResolvePlugin = {
 };
 
 /**
- * Creates a template html file in the Sandcastle app listing the gallery of demos
- * @param {boolean} [noDevelopmentGallery=false] true if the development gallery should not be included in the list
- * @returns {Promise<any>}
+ * Parses Sandcastle config file and returns its values.
+ * @returns {Promise<Record<string,any>>} A promise that resolves to the config values.
  */
-export async function createGalleryList(noDevelopmentGallery) {
-  const configPath = path.join(
-    import.meta.url,
-    "../../packages/sandcastle/sandcastle.config.js",
-  );
-  const config = await import(configPath);
-  const { root: rootDirectory, gallery, sourceUrl } = config.default;
+export async function getSandcastleConfig() {
+  const configPath = "packages/sandcastle/sandcastle.config.js";
+  const configImportPath = path.join(projectRoot, configPath);
+  const config = await import(configImportPath);
+  const options = config.default;
+  return {
+    ...options,
+    configPath,
+  };
+}
+
+/**
+ * Indexes Sandcastle gallery files and writes gallery files to the configured Sandcastle output directory.
+ * @param {boolean} [includeDevelopment=true] true if gallery items flagged as development should be included.
+ * @returns {Promise<void>} A promise that resolves once the gallery files have been indexed and written.
+ */
+export async function buildSandcastleGallery(includeDevelopment) {
+  const { configPath, root, gallery, sourceUrl } = await getSandcastleConfig();
+
+  // Use an absolute path to avoid any descrepency between working directories
+  // All other directories will be relative to the specified root directory
+  const rootDirectory = path.join(path.dirname(configPath), root);
 
   // Paths are specified relative to the config file
   const {
@@ -627,10 +645,12 @@ export async function createGalleryList(noDevelopmentGallery) {
     metadata,
   } = gallery ?? {};
 
-  // Import asynchronously for now while this script is excluded from the release zip
-  const { buildGalleryList } = await import(
-    "../packages/sandcastle/scripts/buildGallery.js"
+  // Import asynchronously, for now, because this following script is not included in the release zip; However, this script will not be run from the release zip
+  const buildGalleryScriptPath = path.join(
+    __dirname,
+    "../packages/sandcastle/scripts/buildGallery.js",
   );
+  const { buildGalleryList } = await import(buildGalleryScriptPath);
 
   await buildGalleryList({
     rootDirectory,
@@ -641,8 +661,17 @@ export async function createGalleryList(noDevelopmentGallery) {
     searchOptions,
     defaultFilters,
     metadata,
-    includeDevelopment: !noDevelopmentGallery,
+    includeDevelopment,
   });
+}
+
+/**
+ * Creates a template html file in the Sandcastle app listing the gallery of demos
+ * @param {boolean} [noDevelopmentGallery=false] true if the development gallery should not be included in the list
+ * @returns {Promise<any>}
+ */
+export async function createGalleryList(noDevelopmentGallery) {
+  await buildSandcastleGallery(!noDevelopmentGallery);
 
   const demoObjects = [];
   const demoJSONs = [];
@@ -655,7 +684,8 @@ export async function createGalleryList(noDevelopmentGallery) {
 
   // In CI, the version is set to something like '1.43.0-branch-name-buildNumber'
   // We need to extract just the Major.Minor version
-  const majorMinor = packageJson.version.match(/^(.*)\.(.*)\./);
+  const version = await getVersion();
+  const majorMinor = version.match(/^(.*)\.(.*)\./);
   const major = majorMinor[1];
   const minor = Number(majorMinor[2]) - 1; // We want the last release, not current release
   const tagVersion = `${major}.${minor}`;
@@ -887,6 +917,7 @@ export async function bundleTestWorkers(options) {
  * @returns
  */
 export async function createIndexJs(workspace) {
+  const version = await getVersion();
   let contents = `globalThis.CESIUM_VERSION = "${version}";\n`;
 
   // Iterate over all provided source files for the workspace and export the assignment based on file name.

server.js

@@ -18,6 +18,8 @@ import {
   glslToJavaScript,
   createIndexJs,
   buildCesium,
+  getSandcastleConfig,
+  buildSandcastleGallery,
 } from "./scripts/build.js";
 
 const argv = yargs(process.argv)
@@ -80,6 +82,22 @@ async function generateDevelopmentBuild() {
   return contexts;
 }
 
+// Delay execution of the callback until a short time has elapsed since it was last invoked, preventing
+// calls to the same function in quick succession from triggering multiple builds.
+const throttleDelay = 500;
+const throttle = (callback) => {
+  let timeout;
+  return () =>
+    new Promise((resolve) => {
+      if (timeout) {
+        clearTimeout(timeout);
+      }
+      timeout = setTimeout(() => {
+        resolve(callback());
+      }, throttleDelay);
+    });
+};
+
 (async function () {
   const gzipHeader = Buffer.from("1F8B08", "hex");
   const production = argv.production;
@@ -267,21 +285,30 @@ async function generateDevelopmentBuild() {
       specsCache.clear();
     });
 
-    const galleryDirectory = "packages/sandcastle/gallery";
-    const galleryWatcher = chokidar.watch([galleryDirectory], {
-      ignored: (file, stats) =>
-        !!stats?.isFile() && !file.endsWith(".yml") && !file.endsWith(".yaml"),
-      ignoreInitial: true,
-    });
     if (!production) {
-      const { buildGalleryList } = await import(
-        "./packages/sandcastle/scripts/buildGallery.js"
+      const { configPath, root, gallery } = await getSandcastleConfig();
+      const baseDirectory = path.relative(root, path.dirname(configPath));
+      const galleryFiles = gallery.files.map((pattern) =>
+        path.join(baseDirectory, pattern),
       );
-      galleryWatcher.on("all", async (event) => {
-        if (event === "add" || event === "change" || event === "unlink") {
-          await buildGalleryList(galleryDirectory);
-        }
+      const galleryWatcher = chokidar.watch(galleryFiles, {
+        ignoreInitial: true,
       });
+
+      galleryWatcher.on(
+        "all",
+        throttle(async () => {
+          const startTime = performance.now();
+          try {
+            await buildSandcastleGallery();
+            console.log(
+              `Gallery built in ${formatTimeSinceInSeconds(startTime)} seconds.`,
+            );
+          } catch (e) {
+            console.error(e);
+          }
+        }),
+      );
     }
 
     // Rebuild jsHintOptions as needed and serve as-is