Merge pull request #4291 from asgerf/js/lean-dependency-installation-plainjava

Approved by erik-krogh

Author: codeql-ci

javascript/extractor/lib/typescript/src/main.ts

@@ -95,11 +95,85 @@ class State {
 
     /** Next response to be delivered. */
     public pendingResponse: string = null;
+
+    /** Map from `package.json` files to their contents. */
+    public parsedPackageJson = new Map<string, any>();
+
+    /** Map from `package.json` files to the file referenced in its `types` or `typings` field. */
+    public packageTypings = new Map<string, string | undefined>();
+
+    /** Map from file path to the enclosing `package.json` file, if any. Will not traverse outside node_modules. */
+    public enclosingPackageJson = new Map<string, string | undefined>();
 }
 let state = new State();
 
 const reloadMemoryThresholdMb = getEnvironmentVariable("SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD", Number, 1000);
 
+function getPackageJson(file: string): any {
+    let cache = state.parsedPackageJson;
+    if (cache.has(file)) return cache.get(file);
+    let result = getPackageJsonRaw(file);
+    cache.set(file, result);
+    return result;
+}
+
+function getPackageJsonRaw(file: string): any {
+    if (!ts.sys.fileExists(file)) return undefined;
+    try {
+        let json = JSON.parse(ts.sys.readFile(file));
+        if (typeof json !== 'object') return undefined;
+        return json;
+    } catch (e) {
+        return undefined;
+    }
+}
+
+function getPackageTypings(file: string): string | undefined {
+    let cache = state.packageTypings;
+    if (cache.has(file)) return cache.get(file);
+    let result = getPackageTypingsRaw(file);
+    cache.set(file, result);
+    return result;
+}
+
+function getPackageTypingsRaw(packageJsonFile: string): string | undefined {
+    let json = getPackageJson(packageJsonFile);
+    if (json == null) return undefined;
+    let typings = json.types || json.typings; // "types" and "typings" are aliases
+    if (typeof typings !== 'string') return undefined;
+    let absolutePath = pathlib.join(pathlib.dirname(packageJsonFile), typings);
+    if (ts.sys.directoryExists(absolutePath)) {
+        absolutePath = pathlib.join(absolutePath, 'index.d.ts');
+    } else if (!absolutePath.endsWith('.ts')) {
+        absolutePath += '.d.ts';
+    }
+    if (!ts.sys.fileExists(absolutePath)) return undefined;
+    return ts.sys.resolvePath(absolutePath);
+}
+
+function getEnclosingPackageJson(file: string): string | undefined {
+    let cache = state.packageTypings;
+    if (cache.has(file)) return cache.get(file);
+    let result = getEnclosingPackageJsonRaw(file);
+    cache.set(file, result);
+    return result;
+}
+
+function getEnclosingPackageJsonRaw(file: string): string | undefined {
+    let packageJson = pathlib.join(file, 'package.json');
+    if (ts.sys.fileExists(packageJson)) {
+        return packageJson;
+    }
+    if (pathlib.basename(file) === 'node_modules') {
+        return undefined;
+    }
+    let dirname = pathlib.dirname(file);
+    if (dirname.length < file.length) {
+        return getEnclosingPackageJson(dirname);
+    }
+    return undefined;
+}
+
 /**
  * Debugging method for finding cycles in the TypeScript AST. Should not be used in production.
  *
@@ -505,14 +579,18 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
     // inverse mapping, nor a way to enumerate all known module names. So we discover all
     // modules on the type roots (usually "node_modules/@types" but this is configurable).
     let typeRoots = ts.getEffectiveTypeRoots(config.options, {
-        directoryExists: (path) => fs.existsSync(path),
+        directoryExists: (path) => ts.sys.directoryExists(path),
         getCurrentDirectory: () => basePath,
     });
 
     for (let typeRoot of typeRoots || []) {
-        if (fs.existsSync(typeRoot) && fs.statSync(typeRoot).isDirectory()) {
+        if (ts.sys.directoryExists(typeRoot)) {
             traverseTypeRoot(typeRoot, "");
         }
+        let virtualTypeRoot = virtualSourceRoot.toVirtualPathIfDirectoryExists(typeRoot);
+        if (virtualTypeRoot != null) {
+            traverseTypeRoot(virtualTypeRoot, "");
+        }
     }
 
     for (let sourceFile of program.getSourceFiles()) {
@@ -549,22 +627,25 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
             if (sourceFile == null) {
                 continue;
             }
-            addModuleBindingFromRelativePath(sourceFile, importPrefix, child);
+            let importPath = getImportPathFromFileInFolder(importPrefix, child);
+            addModuleBindingFromImportPath(sourceFile, importPath);
         }
     }
 
+    function getImportPathFromFileInFolder(folder: string, baseName: string) {
+        let stem = getStem(baseName);
+        return (stem === "index")
+            ? folder
+            : joinModulePath(folder, stem);
+    }
+
     /**
      * Emits module bindings for a module with relative path `folder/baseName`.
      */
-    function addModuleBindingFromRelativePath(sourceFile: ts.SourceFile, folder: string, baseName: string) {
+    function addModuleBindingFromImportPath(sourceFile: ts.SourceFile, importPath: string) {
         let symbol = typeChecker.getSymbolAtLocation(sourceFile);
         if (symbol == null) return; // Happens if the source file is not a module.
 
-        let stem = getStem(baseName);
-        let importPath = (stem === "index")
-            ? folder
-            : joinModulePath(folder, stem);
-
         let canonicalSymbol = getEffectiveExportTarget(symbol); // Follow `export = X` declarations.
         let symbolId = state.typeTable.getSymbolId(canonicalSymbol);
 
@@ -576,7 +657,7 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
         // Note: the `globalExports` map is stored on the original symbol, not the target of `export=`.
         if (symbol.globalExports != null) {
             symbol.globalExports.forEach((global: ts.Symbol) => {
-              state.typeTable.addGlobalMapping(symbolId, global.name);
+                state.typeTable.addGlobalMapping(symbolId, global.name);
             });
         }
     }
@@ -605,11 +686,30 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
         let fullPath = sourceFile.fileName;
         let index = fullPath.lastIndexOf('/node_modules/');
         if (index === -1) return;
+
         let relativePath = fullPath.substring(index + '/node_modules/'.length);
+
         // Ignore node_modules/@types folders here as they are typically handled as type roots.
         if (relativePath.startsWith("@types/")) return;
+
+        // If the enclosing package has a "typings" field, only add module bindings for that file.
+        let packageJsonFile = getEnclosingPackageJson(fullPath);
+        if (packageJsonFile != null) {
+            let json = getPackageJson(packageJsonFile);
+            let typings = getPackageTypings(packageJsonFile);
+            if (json != null && typings != null) {
+                let name = json.name;
+                if (typings === fullPath && typeof name === 'string') {
+                    addModuleBindingFromImportPath(sourceFile, name);
+                } else if (typings != null) {
+                    return; // Typings field prevents access to other files in package.
+                }
+            }
+        }
+
+        // Add module bindings relative to package directory.
         let { dir, base } = pathlib.parse(relativePath);
-        addModuleBindingFromRelativePath(sourceFile, dir, base);
+        addModuleBindingFromImportPath(sourceFile, getImportPathFromFileInFolder(dir, base));
     }
 
     /**

javascript/extractor/lib/typescript/src/virtual_source_root.ts

@@ -55,4 +55,15 @@ export class VirtualSourceRoot {
     }
     return null;
   }
+
+  /**
+   * Maps a path under the real source root to the corresponding path in the virtual source root.
+   */
+  public toVirtualPathIfDirectoryExists(path: string) {
+    let virtualPath = this.toVirtualPath(path);
+    if (virtualPath != null && ts.sys.directoryExists(virtualPath)) {
+        return virtualPath;
+    }
+    return null;
+  }
 }

javascript/extractor/src/com/semmle/js/dependencies/AsyncFetcher.java

@@ -0,0 +1,122 @@
+package com.semmle.js.dependencies;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+import com.semmle.js.dependencies.packument.Packument;
+
+/**
+ * Asynchronous I/O operations needed for dependency installation.
+ * <p>
+ * The methods in this class are non-blocking, that is, they return more or less immediately, always scheduling the work
+ * in the provided executor service. Requests are cached where it makes sense.
+ */
+public class AsyncFetcher {
+    private Fetcher fetcher = new Fetcher();
+    private ExecutorService executor;
+    private Consumer<CompletionException> errorReporter;
+
+    /**
+     * @param executor thread pool to perform I/O tasks
+     * @param errorReporter called once for each error from the underlying I/O tasks
+     */
+    public AsyncFetcher(ExecutorService executor, Consumer<CompletionException> errorReporter) {
+        this.executor = executor;
+        this.errorReporter = errorReporter;
+    }
+
+    private CompletionException makeError(String message, Exception cause) {
+        CompletionException ex = new CompletionException(message, cause);
+        errorReporter.accept(ex); // Handle here to ensure each exception is logged at most once, not once per consumer
+        throw ex;
+    }
+
+    private class CachedOperation<K, V> {
+        private Map<K, CompletableFuture<V>> cache = new LinkedHashMap<>();
+
+        public synchronized CompletableFuture<V> get(K key, Supplier<V> builder) {
+            CompletableFuture<V> future = cache.get(key);
+            if (future == null) {
+                future = CompletableFuture.supplyAsync(() -> builder.get(), executor);
+                cache.put(key, future);
+            }
+            return future;
+        }
+    }
+
+    private CachedOperation<String, Packument> packuments = new CachedOperation<>();
+
+    /**
+     * Returns a future that completes with the packument for the given package.
+     * <p>
+     * At most one fetch will be performed.
+     */
+    public CompletableFuture<Packument> getPackument(String packageName) {
+        return packuments.get(packageName, () -> {
+            try {
+                return fetcher.getPackument(packageName);
+            } catch (IOException e) {
+                throw makeError("Could not fetch packument for " + packageName, e);
+            }
+        });
+    }
+
+    /** Result of a tarball extraction */
+    private static class ExtractionResult {
+        /** The directory into which the tarball was extracted. */
+        Path destDir;
+
+        /** Files created by the extraction, relative to <code>destDir</code>. */
+        List<Path> relativePaths;
+
+        ExtractionResult(Path destDir, List<Path> relativePaths) {
+            this.destDir = destDir;
+            this.relativePaths = relativePaths;
+        }
+    }
+
+    private CachedOperation<String, ExtractionResult> tarballExtractions = new CachedOperation<>();
+
+    /**
+     * Extracts the relevant contents of the given tarball URL in the given folder;
+     * the returned future completes when done.
+     *
+     * If the same tarball has already been extracted elsewhere, then symbolic links are added to `destDir` linking to the already extracted tarball. 
+     */
+    public CompletableFuture<Void> installFromTarballUrl(String tarballUrl, Path destDir) {
+        return tarballExtractions.get(tarballUrl, () -> {
+            try {
+                List<Path> relativePaths = fetcher.extractFromTarballUrl(tarballUrl, destDir);
+                return new ExtractionResult(destDir, relativePaths);
+            } catch (IOException e) {
+                throw makeError("Could not install package from " + tarballUrl, e);
+            }
+        }).thenAccept(extractionResult -> {
+            if (!extractionResult.destDir.equals(destDir)) {
+                // We've been asked to extract the same tarball into multiple directories (due to multiple package.json files).
+                // Symlink files from the original directory instead of extracting again.
+                // In principle we could symlink the whole directory, but directory symlinks are hard to create in a portable way.
+                System.out.println("Creating symlink farm from " + destDir + " to " + extractionResult.destDir);
+                for (Path relativePath : extractionResult.relativePaths) {
+                    Path originalFile = extractionResult.destDir.resolve(relativePath);
+                    Path newFile = destDir.resolve(relativePath);
+                    try {
+                        fetcher.mkdirp(newFile.getParent());
+                        Files.createSymbolicLink(newFile, originalFile);
+                    } catch (IOException e) {
+                        throw makeError("Failed to create symlink " + newFile + " -> " + originalFile, e);
+                    }
+                }
+            }
+        });
+    }
+}