Merge pull request #4068 from github/koesie10/expand-short-paths

Add expansion of short paths using native Windows call

Author: koesie10

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -4,6 +4,7 @@ import {
   mkdirs,
   readdir,
   unlinkSync,
+  rename,
   remove,
   writeFile,
 } from "fs-extra";
@@ -45,6 +46,10 @@ async function copyPackage(
       copyDirectory(resolve(sourcePath, file), resolve(destPath, file)),
     ),
   );
+
+  // The koffi directory needs to be located at the root of the package to ensure
+  // that the koffi package can find its native modules.
+  await rename(resolve(destPath, "out", "koffi"), resolve(destPath, "koffi"));
 }
 
 export async function deployPackage(): Promise<DeployedPackage> {

extensions/ql-vscode/gulpfile.ts/index.ts

@@ -5,7 +5,7 @@ import {
   checkTypeScript,
   watchCheckTypeScript,
   cleanOutput,
-  copyWasmFiles,
+  copyModules,
 } from "./typescript";
 import { compileTextMateGrammar } from "./textmate";
 import { packageExtension } from "./package";
@@ -21,7 +21,7 @@ export const buildWithoutPackage = series(
   cleanOutput,
   parallel(
     compileEsbuild,
-    copyWasmFiles,
+    copyModules,
     checkTypeScript,
     compileTextMateGrammar,
     compileViewEsbuild,
@@ -46,7 +46,7 @@ export {
   watchCheckTypeScript,
   watchViewEsbuild,
   compileEsbuild,
-  copyWasmFiles,
+  copyModules,
   checkTypeScript,
   injectAppInsightsKey,
   compileViewEsbuild,

extensions/ql-vscode/gulpfile.ts/typescript.ts

@@ -1,5 +1,5 @@
 import { gray, red } from "ansi-colors";
-import { dest, src, watch } from "gulp";
+import { dest, parallel, src, watch } from "gulp";
 import esbuild from "gulp-esbuild";
 import type { reporter } from "gulp-typescript";
 import { createProject } from "gulp-typescript";
@@ -71,7 +71,7 @@ export function watchCheckTypeScript() {
   watch(["src/**/*.ts", "!src/view/**/*.ts"], checkTypeScript);
 }
 
-export function copyWasmFiles() {
+function copyWasmFiles() {
   // We need to copy this file for the source-map package to work. Without this fie, the source-map
   // package is not able to load the WASM file because we are not including the full node_modules
   // directory. In version 0.7.4, it is not possible to call SourceMapConsumer.initialize in Node environments
@@ -83,3 +83,18 @@ export function copyWasmFiles() {
     encoding: false,
   }).pipe(dest("out"));
 }
+
+function copyNativeAddonFiles() {
+  // We need to copy these files manually because we only want to include Windows x64 to limit
+  // the size of the extension. Windows x64 is the most common platform that requires short path
+  // expansion, so we only include this platform.
+  // See src/common/short-paths.ts
+  return pipeline(
+    src("node_modules/koffi/build/koffi/win32_x64/*.node", {
+      encoding: false,
+    }),
+    dest("out/koffi/win32_x64"),
+  );
+}
+
+export const copyModules = parallel(copyWasmFiles, copyNativeAddonFiles);

extensions/ql-vscode/package-lock.json

@@ -1995,6 +1995,7 @@
     "d3-graphviz": "^5.0.2",
     "fs-extra": "^11.1.1",
     "js-yaml": "^4.1.0",
+    "koffi": "^2.12.0",
     "msw": "^2.7.4",
     "nanoid": "^5.0.7",
     "p-queue": "^8.0.1",

extensions/ql-vscode/package.json

@@ -1,7 +1,9 @@
-import { platform } from "os";
+import { arch, platform } from "os";
 import { basename, dirname, join, normalize, resolve } from "path";
 import { lstat, readdir } from "fs/promises";
 import type { BaseLogger } from "./logging";
+import type { KoffiFunction } from "koffi";
+import { getErrorMessage } from "./helpers-pure";
 
 /**
  * Expands a path that potentially contains 8.3 short names (e.g. "C:\PROGRA~1" instead of "C:\Program Files").
@@ -32,7 +34,23 @@ export async function expandShortPaths(
     return absoluteShortPath;
   }
 
-  return await expandShortPathRecursive(absoluteShortPath, logger);
+  const longPath = await expandShortPathRecursive(absoluteShortPath, logger);
+  if (longPath.indexOf("~") < 0) {
+    return longPath;
+  }
+
+  void logger.log(
+    "Short path was not resolved to long path, using native method",
+  );
+
+  try {
+    return await expandShortPathNative(absoluteShortPath, logger);
+  } catch (e: unknown) {
+    void logger.log(
+      `Failed to expand short path using native method: ${getErrorMessage(e)}`,
+    );
+    return longPath;
+  }
 }
 
 /**
@@ -115,3 +133,46 @@ async function expandShortPathRecursive(
   const longBase = await expandShortPathComponent(dir, shortBase, logger);
   return join(dir, longBase);
 }
+
+let GetLongPathNameW: KoffiFunction | undefined;
+
+async function expandShortPathNative(shortPath: string, logger: BaseLogger) {
+  if (platform() !== "win32") {
+    throw new Error("expandShortPathNative is only supported on Windows");
+  }
+
+  if (arch() !== "x64") {
+    throw new Error(
+      "expandShortPathNative is only supported on x64 architecture",
+    );
+  }
+
+  if (GetLongPathNameW === undefined) {
+    // We are using koffi/indirect here to avoid including the native addon for all
+    // platforms in the bundle since this is only used on Windows. Instead, the
+    // native addon is included in the Gulpfile.
+    const koffi = await import("koffi/indirect");
+
+    const lib = koffi.load("kernel32.dll");
+    GetLongPathNameW = lib.func("__stdcall", "GetLongPathNameW", "uint32", [
+      "str16",
+      "str16",
+      "uint32",
+    ]);
+  }
+
+  const MAX_PATH = 32767;
+  const buffer = Buffer.alloc(MAX_PATH * 2, 0);
+
+  const result = GetLongPathNameW(shortPath, buffer, MAX_PATH);
+
+  if (result === 0) {
+    throw new Error("Failed to get long path name");
+  }
+
+  const longPath = buffer.toString("utf16le", 0, (result - 1) * 2);
+
+  void logger.log(`Expanded short path ${shortPath} to ${longPath}`);
+
+  return longPath;
+}

extensions/ql-vscode/src/common/short-paths.ts

@@ -0,0 +1,4 @@
+// koffi/indirect is untyped in the upstream package, but it exports the same functions as koffi.
+declare module "koffi/indirect" {
+  export * from "koffi";
+}

extensions/ql-vscode/src/koffi.d.ts

@@ -2,9 +2,8 @@ import type { CancellationToken } from "vscode";
 import { Uri, window } from "vscode";
 import { join, sep, basename, relative } from "path";
 import { dump, load } from "js-yaml";
-import { copy, writeFile, readFile, mkdirp } from "fs-extra";
-import type { DirectoryResult } from "tmp-promise";
-import { dir, tmpName } from "tmp-promise";
+import { copy, writeFile, readFile, mkdirp, remove } from "fs-extra";
+import { nanoid } from "nanoid";
 import { tmpDir } from "../tmp-dir";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import type { Credentials } from "../common/authentication";
@@ -236,39 +235,28 @@ async function copyExistingQueryPack(
 }
 
 interface RemoteQueryTempDir {
-  remoteQueryDir: DirectoryResult;
+  remoteQueryDir: string;
   queryPackDir: string;
   compiledPackDir: string;
   bundleFile: string;
 }
 
 async function createRemoteQueriesTempDirectory(): Promise<RemoteQueryTempDir> {
-  const shortRemoteQueryDir = await dir({
-    dir: tmpDir.name,
-    unsafeCleanup: true,
-  });
   // Expand 8.3 filenames here to work around a CLI bug where `codeql pack bundle` produces an empty
   // archive if the pack path contains any 8.3 components.
-  const remoteQueryDir = {
-    ...shortRemoteQueryDir,
-    path: await expandShortPaths(shortRemoteQueryDir.path, extLogger),
-  };
-  const queryPackDir = join(remoteQueryDir.path, "query-pack");
+  const tmpDirPath = await expandShortPaths(tmpDir.name, extLogger);
+
+  const remoteQueryDir = join(tmpDirPath, `remote-query-${nanoid()}`);
+  await mkdirp(remoteQueryDir);
+
+  const queryPackDir = join(remoteQueryDir, "query-pack");
   await mkdirp(queryPackDir);
-  const compiledPackDir = join(remoteQueryDir.path, "compiled-pack");
-  const bundleFile = await expandShortPaths(
-    await getPackedBundlePath(tmpDir.name),
-    extLogger,
-  );
-  return { remoteQueryDir, queryPackDir, compiledPackDir, bundleFile };
-}
 
-async function getPackedBundlePath(remoteQueryDir: string): Promise<string> {
-  return tmpName({
-    dir: remoteQueryDir,
-    postfix: "generated.tgz",
-    prefix: "qlpack",
-  });
+  const compiledPackDir = join(remoteQueryDir, "compiled-pack");
+
+  const bundleFile = join(remoteQueryDir, `qlpack-${nanoid()}-generated.tgz`);
+
+  return { remoteQueryDir, queryPackDir, compiledPackDir, bundleFile };
 }
 
 interface PreparedRemoteQuery {
@@ -337,7 +325,7 @@ export async function prepareRemoteQueryRun(
       token,
     );
   } finally {
-    await tempDir.remoteQueryDir.cleanup();
+    await remove(tempDir.remoteQueryDir);
   }
 
   if (token.isCancellationRequested) {