Merge rc/1.21 into master

Author: markshannon

javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java

@@ -33,14 +33,21 @@
 import java.io.OutputStreamWriter;
 import java.lang.ProcessBuilder.Redirect;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 /**
  * The Java half of our wrapper for invoking the TypeScript parser.
  *
  * <p>The Node.js half of the wrapper is expected to live at {@code
  * $SEMMLE_DIST/tools/typescript-parser-wrapper/main.js}; non-standard locations can be configured
- * using the property {@link #PARSER_WRAPPER_PATH_ENV_VAR}.
+ * using the property {@value #PARSER_WRAPPER_PATH_ENV_VAR}.
+ * 
+ * <p>The script launches the Node.js wrapper in the Node.js runtime, looking for {@code node}
+ * on the {@code PATH} by default. Non-standard locations can be configured using the property
+ * {@value #TYPESCRIPT_NODE_RUNTIME_VAR}, and additional arguments can be configured using the
+ * property {@value #TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS_VAR}.
  *
  * <p>The script is started upon parsing the first TypeScript file and then is kept running in the
  * background, passing it requests for parsing files and getting JSON-encoded ASTs as responses.
@@ -52,6 +59,18 @@ public class TypeScriptParser {
    */
   public static final String PARSER_WRAPPER_PATH_ENV_VAR = "SEMMLE_TYPESCRIPT_PARSER_WRAPPER";
 
+  /**
+   * An environment variable that can be set to indicate the location of the Node.js runtime,
+   * as an alternative to adding Node to the PATH.
+   */
+  public static final String TYPESCRIPT_NODE_RUNTIME_VAR = "SEMMLE_TYPESCRIPT_NODE_RUNTIME";
+
+  /**
+   * An environment variable that can be set to provide additional arguments to the Node.js runtime
+   * each time it is invoked. Arguments should be separated by spaces.
+   */
+  public static final String TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS_VAR = "SEMMLE_TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS";
+
   /**
    * An environment variable that can be set to specify a timeout to use when verifying the
    * TypeScript installation, in milliseconds. Default is 10000.
@@ -91,6 +110,15 @@ public class TypeScriptParser {
 
   private String nodeJsVersionString;
 
+  /** Command to launch the Node.js runtime. Initialised by {@link #verifyNodeInstallation}. */
+  private String nodeJsRuntime;
+
+  /**
+   * Arguments to pass to the Node.js runtime each time it is invoked.
+   * Initialised by {@link #verifyNodeInstallation}.
+   */
+  private List<String> nodeJsRuntimeExtraArgs = Collections.emptyList();
+
   /** If non-zero, we use this instead of relying on the corresponding environment variable. */
   private int typescriptRam = 0;
 
@@ -102,12 +130,16 @@ public void setTypescriptRam(int megabytes) {
   /**
    * Verifies that Node.js and TypeScript are installed and throws an exception otherwise.
    *
-   * @param verbose if true, log the version strings and NODE_PATH.
+   * @param verbose if true, log the Node.js executable path, version strings, and any additional arguments.
    */
   public void verifyInstallation(boolean verbose) {
     verifyNodeInstallation();
     if (verbose) {
+      System.out.println("Found Node.js at: " + nodeJsRuntime);
       System.out.println("Found Node.js version: " + nodeJsVersionString);
+      if (!nodeJsRuntimeExtraArgs.isEmpty()) {
+        System.out.println("Additional arguments for Node.js: " + nodeJsRuntimeExtraArgs);
+      }
     }
   }
 
@@ -117,7 +149,24 @@ public String verifyNodeInstallation() {
 
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     ByteArrayOutputStream err = new ByteArrayOutputStream();
-    Builder b = new Builder(out, err, getParserWrapper().getParentFile(), "node", "--version");
+    
+    // Determine where to find the Node.js runtime.
+    String explicitNodeJsRuntime = Env.systemEnv().get(TYPESCRIPT_NODE_RUNTIME_VAR);
+    if (explicitNodeJsRuntime != null) {
+      // Use the specified Node.js executable.
+      nodeJsRuntime = explicitNodeJsRuntime;
+    } else {
+      // Look for `node` on the PATH.
+      nodeJsRuntime = "node";
+    }
+
+    // Determine any additional arguments to be passed to Node.js each time it's called.
+    String extraArgs = Env.systemEnv().get(TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS_VAR);
+    if (extraArgs != null) {
+      nodeJsRuntimeExtraArgs = Arrays.asList(extraArgs.split("\\s+"));
+    }
+
+    Builder b = new Builder(getNodeJsRuntimeInvocation("--version"), out, err, getParserWrapper().getParentFile());
     b.expectFailure(); // We want to do our own logging in case of an error.
 
     int timeout = Env.systemEnv().getInt(TYPESCRIPT_TIMEOUT_VAR, 10000);
@@ -144,6 +193,21 @@ public String verifyNodeInstallation() {
     }
   }
 
+  /**
+   * Gets a command line to invoke the Node.js runtime.
+   * Any arguments in {@link TypeScriptParser#nodeJsRuntimeExtraArgs}
+   * are passed first, followed by those in {@code args}.
+   */
+  private List<String> getNodeJsRuntimeInvocation(String ...args) {
+    List<String> result = new ArrayList<>();
+    result.add(nodeJsRuntime);
+    result.addAll(nodeJsRuntimeExtraArgs);
+    for(String arg : args) {
+      result.add(arg);
+    }
+    return result;
+  }
+
   private static int getMegabyteCountFromPrefixedEnv(String suffix, int defaultValue) {
     String envVar = "SEMMLE_" + suffix;
     String value = Env.systemEnv().get(envVar);
@@ -172,10 +236,11 @@ private void setupParserWrapper() {
     int reserveMemoryMb = getMegabyteCountFromPrefixedEnv(TYPESCRIPT_RAM_RESERVE_SUFFIX, 400);
 
     File parserWrapper = getParserWrapper();
-    List<String> cmd = new ArrayList<>();
-    cmd.add("node");
-    cmd.add("--max_old_space_size=" + (mainMemoryMb + reserveMemoryMb));
-    cmd.add(parserWrapper.getAbsolutePath());
+    
+    List<String> cmd = getNodeJsRuntimeInvocation(
+        "--max_old_space_size=" + (mainMemoryMb + reserveMemoryMb),
+        parserWrapper.getAbsolutePath()
+    );
     ProcessBuilder pb = new ProcessBuilder(cmd);
     parserWrapperCommand = StringUtil.glue(" ", cmd);
     pb.environment().put("SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD", "" + mainMemoryMb);

python/ql/src/semmle/python/objects/Callables.qll

@@ -161,6 +161,8 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
 
     override predicate contextSensitiveCallee() { any() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 
@@ -284,6 +286,8 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** Class representing methods of built-in classes (otherwise known as method-descriptors) such as `list.append`.
@@ -376,6 +380,8 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** Class representing bound-methods.
@@ -461,6 +467,8 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
         function = this.getFunction() and offset = 1
     }
 
+    override predicate useOriginAsLegacyObject() { any() }
+
     override predicate contextSensitiveCallee() {
         this.getFunction().contextSensitiveCallee()
     }

python/ql/src/semmle/python/objects/Classes.qll

@@ -92,6 +92,8 @@ abstract class ClassObjectInternal extends ObjectInternal {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /* Classes aren't usually iterable, but can e.g. Enums */
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 

python/ql/src/semmle/python/objects/Constants.qll

@@ -71,6 +71,8 @@ abstract class ConstantObjectInternal extends ObjectInternal {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 private abstract class BooleanObjectInternal extends ConstantObjectInternal {

python/ql/src/semmle/python/objects/Descriptors.qll

@@ -93,6 +93,8 @@ class PropertyInternal extends ObjectInternal, TProperty {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /* Properties aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
@@ -183,6 +185,8 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /* Classmethods aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
@@ -259,6 +263,8 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /* Staticmethods aren't iterable */
     override ObjectInternal getIterNext() { none() }
 

python/ql/src/semmle/python/objects/Instances.qll

@@ -164,6 +164,8 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject {
         )
     }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** A class representing context-free instances represented by `self` in the source code
@@ -266,6 +268,8 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject {
         this.getClass().attribute("__init__", init, _)
     }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** A class representing a value that has a known class, but no other information */
@@ -372,6 +376,8 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { any() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }
@@ -482,6 +488,8 @@ class SuperInstance extends TSuperInstance, ObjectInternal {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { any() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }

python/ql/src/semmle/python/objects/Modules.qll

@@ -54,6 +54,8 @@ abstract class ModuleObjectInternal extends ObjectInternal {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /* Modules aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
@@ -313,10 +315,6 @@ class AbsentModuleObjectInternal extends ModuleObjectInternal, TAbsentModule {
         none()
     }
 
-    override predicate isMissing() {
-        any()
-    }
-
 }
 
 /** A class representing an attribute of a missing module. */
@@ -402,15 +400,13 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA
 
     override predicate subscriptUnknown() { any() }
 
-    override predicate isMissing() {
-        any()
-    }
-
     /* We know what this is called, but not its innate name */
     override string getName() { none() }
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /* Modules aren't iterable */
     override ObjectInternal getIterNext() { none() }
 

python/ql/src/semmle/python/objects/ObjectAPI.qll

@@ -78,14 +78,6 @@ class Value extends TObject {
         this.(ObjectInternal).isBuiltin()
     }
 
-    /** Holds if this value represents an entity that is inferred to exist,
-     * but missing from the database.
-     * Most commonly, this is a module that is imported, but wasn't present during extraction.
-     */
-    predicate isMissing() {
-        this.(ObjectInternal).isMissing()
-    }
-
     predicate hasLocationInfo(string filepath, int bl, int bc, int el, int ec) {
         this.(ObjectInternal).getOrigin().getLocation().hasLocationInfo(filepath, bl, bc, el, ec)
         or

python/ql/src/semmle/python/objects/ObjectInternal.qll

@@ -155,11 +155,12 @@ class ObjectInternal extends TObject {
      */
     predicate functionAndOffset(CallableObjectInternal function, int offset) { none() }
 
-    /** Holds if this 'object' represents an entity that is inferred to exist
-     * but is missing from the database */
-    predicate isMissing() {
-        none()
-    }
+    /** Holds if this 'object' represents an entity that should be exposed to the legacy points_to API
+     * This should hold for almost all objects that do not have an underlying DB object representing their source,
+     * for example `super` objects and bound-method. This should not hold for objects that are inferred to exists by
+     * an import statements or the like, but which aren't in the database. */
+     /* This predicate can be removed when the legacy points_to API is removed. */
+    abstract predicate useOriginAsLegacyObject();
 
     /** Gets the name of this of this object if it has a meaningful name.
      * Note that the name of an object is not necessarily the name by which it is called
@@ -260,6 +261,8 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }
@@ -341,6 +344,8 @@ class UnknownInternal extends ObjectInternal, TUnknown {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }
@@ -421,6 +426,8 @@ class UndefinedInternal extends ObjectInternal, TUndefined {
 
     override string getName() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /** Holds if this object requires context to determine the object resulting from a call to it.
      * True for most callables. */
     override predicate contextSensitiveCallee() { none() }
@@ -608,6 +615,8 @@ class DecoratedFunction extends ObjectInternal, TDecoratedFunction {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** Helper for boolean predicates returning both `true` and `false` */

python/ql/src/semmle/python/objects/Sequences.qll

@@ -121,6 +121,9 @@ class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal {
             result = count(int n | exists(b.getItem(n)))
         )
     }
+
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** A tuple declared by a tuple expression in the Python source code */
@@ -152,6 +155,8 @@ class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal {
         )
     }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** A tuple created by a `*` parameter */
@@ -180,6 +185,9 @@ class VarargsTupleObjectInternal extends TVarargsTuple,  TupleObjectInternal {
     override int length() {
         this = TVarargsTuple(_, _, _, result)
     }
+
+    override predicate useOriginAsLegacyObject() { any() }
+
 }
 
 
@@ -260,4 +268,6 @@ class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectIntern
 
     override predicate functionAndOffset(CallableObjectInternal function, int offset) { none() }
 
+    override predicate useOriginAsLegacyObject() { any() }
+
 }