Merge pull request #1483 from markshannon/merge-121

Merge rc/1.21 into master

Author: taus-semmle

change-notes/support/language-support.rst

@@ -17,5 +17,6 @@ Note that where there are several versions or dialects of a language, the suppor
     .. [2] In addition, support is included for the preview features of C# 8.0 and .NET Core 3.0.
     .. [3] The best results are achieved with COBOL code that stays close to the ANSI 85 standard.  
     .. [4] Builds that execute on Java 6 to 12 can be analyzed. The analysis understands Java 12 language features.
-    .. [5] JSX and Flow code, YAML, JSON, HTML, and XML files may also be analyzed with JavaScript files. 
-    .. [6] TypeScript analysis is performed by running the JavaScript extractor with TypeScript enabled. This is the default for LGTM.   
+    .. [5] ECJ is supported when the build invokes it via the Maven Compiler plugin or the Takari Lifecycle plugin.
+    .. [6] JSX and Flow code, YAML, JSON, HTML, and XML files may also be analyzed with JavaScript files. 
+    .. [7] TypeScript analysis is performed by running the JavaScript extractor with TypeScript enabled. This is the default for LGTM.   

change-notes/support/versions-compilers.csv

@@ -1,5 +1,5 @@
 Language,Variants,Compilers,Extensions
-C/C++,"C89, C99, C11, C++98, C++03, C++11, C++14, C++17","Clang extensions (up to Clang 8.0)
+C/C++,"C89, C99, C11, C++98, C++03, C++11, C++14, C++17","Clang extensions (up to Clang 8.0),
 
 GNU extensions (up to GCC 8.3), 
 
@@ -10,9 +10,9 @@ C#,C# up to 7.3. with .NET up to 4.8 [2]_.,"Microsoft Visual Studio up to 2019,
 
 .NET Core up to 2.2","``.sln``, ``.csproj``, ``.cs``, ``.cshtml``, ``.xaml``"
 COBOL,ANSI 85 or newer [3]_.,Not applicable,"``.cbl``, ``.CBL``, ``.cpy``, ``.CPY``, ``.copy``, ``.COPY``"
-Java,"Java 6 to 12 [4]_.","javac (OpenJDK and Oracle JDK)
+Java,"Java 6 to 12 [4]_.","javac (OpenJDK and Oracle JDK),
 
-Eclipse compiler for Java (ECJ) batch compiler",``.java``
-JavaScript,ECMAScript 2019 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhm``, ``.xhtml``, ``.vue``, ``.json``, ``.yaml``, ``.yml``, ``.raml``, ``.xml`` [5]_."
+Eclipse compiler for Java (ECJ) [5]_.",``.java``
+JavaScript,ECMAScript 2019 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhm``, ``.xhtml``, ``.vue``, ``.json``, ``.yaml``, ``.yml``, ``.raml``, ``.xml`` [6]_."
 Python,"2.7, 3.5, 3.6, 3.7",Not applicable,``.py``
-TypeScript [6]_.,"2.6-3.5",Standard TypeScript compiler,"``.ts``, ``.tsx``"
+TypeScript [7]_.,"2.6-3.5",Standard TypeScript compiler,"``.ts``, ``.tsx``"

python/ql/src/semmle/python/Flow.qll

@@ -985,6 +985,7 @@ class BasicBlock extends @py_flow_node {
 
     /** Dominance frontier of a node x is the set of all nodes `other` such that `this` dominates a predecessor
      * of `other` but does not strictly dominate `other` */
+    pragma[noinline]
     predicate dominanceFrontier(BasicBlock other) {
         this.dominates(other.getAPredecessor()) and not this.strictlyDominates(other)
     }

python/ql/src/semmle/python/Module.qll

@@ -9,9 +9,13 @@ class Module extends Module_, Scope, AstNode {
     override string toString() {
         result = this.getKind() + " " + this.getName()
         or
-        /* No name is defined, which means that this is not on an import path. So it must be a script */
+        /* No name is defined, which means that this module is not on an import path. So it must be a script */
         not exists(this.getName()) and not this.isPackage() and
         result = "Script " + this.getFile().getShortName()
+        or
+        /* Package missing name, so just use the path instead */
+        not exists(this.getName()) and this.isPackage() and
+        result = "Package at "  + this.getPath().getAbsolutePath()
     }
 
     /** This method will be deprecated in the next release. Please use `getEnclosingScope()` instead.
@@ -67,9 +71,9 @@ class Module extends Module_, Scope, AstNode {
     string getAnExport() {
         py_exports(this, result)
         or
-        exists(ModuleValue mod |
+        exists(ModuleObjectInternal mod |
             mod.getSource() = this.getEntryNode() |
-            mod.exports(result)
+            mod.(ModuleValue).exports(result)
         )
     }
 
@@ -191,6 +195,7 @@ class Module extends Module_, Scope, AstNode {
 
 }
 
+
 bindingset[name]
 private predicate legalDottedName(string name) {
     name.regexpMatch("(\\p{L}|_)(\\p{L}|\\d|_)*(\\.(\\p{L}|_)(\\p{L}|\\d|_)*)*")
@@ -240,3 +245,30 @@ private predicate isStubRoot(Folder f) {
     f.getAbsolutePath().matches("%/data/python/stubs")
 }
 
+
+/** Holds if the Container `c` should be the preferred file or folder for
+ * the given name when performing imports.
+ * Trivially true for any container if it is the only one with its name.
+ * However, if there are several modules with the same name, then
+ * this is the module most likely to be imported under that name.
+ */
+predicate isPreferredModuleForName(Container c, string name) {
+    exists(int p |
+        p = min(int x | x = priorityForName(_, name)) and
+        p = priorityForName(c, name)
+    )
+}
+
+private int priorityForName(Container c, string name) {
+    name = moduleNameFromFile(c) and
+    (
+        // In the source
+        exists(c.getRelativePath()) and result = -1
+        or
+        // On an import path
+        exists(c.getImportRoot(result))
+        or
+        // Otherwise
+        result = 10000
+    )
+}

python/ql/src/semmle/python/objects/Callables.qll

@@ -11,9 +11,6 @@ private import semmle.python.types.Builtins
 
 abstract class CallableObjectInternal extends ObjectInternal {
 
-    /** Gets the name of this callable */
-    abstract string getName();
-
     /** Gets the scope of this callable if it has one */
     abstract Function getScope();
 

python/ql/src/semmle/python/objects/Constants.qll

@@ -73,11 +73,6 @@ abstract class ConstantObjectInternal extends ObjectInternal {
 
 private abstract class BooleanObjectInternal extends ConstantObjectInternal {
 
-    BooleanObjectInternal() {
-        this = TTrue() or this = TFalse()
-    }
-
-
     override ObjectInternal getClass() {
         result = TBuiltinClassObject(Builtin::special("bool"))
     }

python/ql/src/semmle/python/objects/Instances.qll

@@ -440,16 +440,21 @@ class SuperInstance extends TSuperInstance, ObjectInternal {
     pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
 
     pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
-        PointsToInternal::attributeRequired(this, name) and
         exists(ObjectInternal cls_attr, CfgOrigin attr_orig |
-            this.lookup(name, cls_attr, attr_orig)
+            this.attribute_descriptor(name, cls_attr, attr_orig)
             |
             cls_attr.isDescriptor() = false and value = cls_attr and origin = attr_orig
             or
             cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this.getSelf(), value, origin)
         )
     }
 
+    /* Helper for `attribute` */
+    pragma [noinline] private predicate attribute_descriptor(string name, ObjectInternal cls_attr, CfgOrigin attr_orig) {
+        PointsToInternal::attributeRequired(this, name) and
+        this.lookup(name, cls_attr, attr_orig)
+    }
+
     private predicate lookup(string name, ObjectInternal value, CfgOrigin origin) {
         Types::getMro(this.getSelf().getClass()).startingAt(this.getStartClass()).getTail().lookup(name, value, origin)
     }

python/ql/src/semmle/python/objects/ObjectAPI.qll

@@ -71,13 +71,6 @@ class Value extends TObject {
         this.(ObjectInternal).attribute(name, result, _)
     }
 
-    /** DEPRECATED: For backwards compatibility with old API
-     * Use `Value` instead of `ObjectSource`.
-     */
-    deprecated ObjectSource getSource() {
-        result = this.(ObjectInternal).getSource()
-    }
-
     /** Holds if this value is builtin. Applies to built-in functions and methods,
      * but also integers and strings.
      */
@@ -153,9 +146,9 @@ module Module {
      *
      * Note that the name used to refer to a module is not
      * necessarily its name. For example,
-     * there are modules refered to by the name `os.path`,
+     * there are modules referred to by the name `os.path`,
      * but that are not named `os.path`, for example the module `posixpath`.
-     * Such that the follwing is true:
+     * Such that the following is true:
      * `Module::named("os.path").getName() = "posixpath"
      */
     ModuleValue named(string name) {

python/ql/src/semmle/python/objects/TObject.qll

@@ -47,12 +47,13 @@ cached newtype TObject =
     or
     /* Package objects */
     TPackageObject(Folder f) {
-        exists(moduleNameFromFile(f))
+        isPreferredModuleForName(f, _)
     }
     or
     /* Python module objects */
     TPythonModule(Module m) {
-        not m.isPackage() and not exists(SyntaxError se | se.getFile() = m.getFile())
+        not m.isPackage() and isPreferredModuleForName(m.getFile(), _) and
+        not exists(SyntaxError se | se.getFile() = m.getFile())
     }
     or
     /* `True` */

python/ql/src/semmle/python/pointsto/PointsTo.qll

@@ -141,7 +141,7 @@ module PointsTo {
         )
         or
         not f.isParameter() and
-        exists(Value value |
+        exists(ObjectInternal value |
             PointsToInternal::pointsTo(f.(DefinitionNode).getValue(), context, value, origin) and
             cls = value.getClass().getSource() |
             obj = value.getSource() or
@@ -151,7 +151,7 @@ module PointsTo {
 
     deprecated predicate
     ssa_variable_points_to(EssaVariable var, PointsToContext context, Object obj, ClassObject cls, CfgOrigin origin) {
-        exists(Value value |
+        exists(ObjectInternal value |
             PointsToInternal::variablePointsTo(var, context, value, origin) and
             cls = value.getClass().getSource() |
             obj = value.getSource()
@@ -160,8 +160,8 @@ module PointsTo {
 
     deprecated
     CallNode get_a_call(Object func, PointsToContext context) {
-        exists(Value value |
-            result = value.getACall(context) and
+        exists(ObjectInternal value |
+            result = value.(Value).getACall(context) and
             func = value.getSource()
         )
     }
@@ -402,7 +402,7 @@ cached module PointsToInternal {
         or
         scope_entry_points_to(def, context, value, origin)
         or
-        InterModulePointsTo::implicit_submodule_points_to(def, context, value, origin)
+        InterModulePointsTo::implicit_submodule_points_to(def, value, origin) and context.isImport()
         or
         iteration_definition_points_to(def, context, value, origin)
         /*
@@ -647,18 +647,22 @@ private module InterModulePointsTo {
         )
     }
 
+    /* Helper for implicit_submodule_points_to */
+    private ModuleObjectInternal getModule(ImplicitSubModuleDefinition def) {
+       exists(PackageObjectInternal package |
+           package.getSourceModule() = def.getDefiningNode().getScope() and 
+           result =  package.submodule(def.getSourceVariable().getName())
+       )
+    }
+
     /** Implicit "definition" of the names of submodules at the start of an `__init__.py` file.
      *
      * PointsTo isn't exactly how the interpreter works, but is the best approximation we can manage statically.
      */
     pragma [noinline]
-    predicate implicit_submodule_points_to(ImplicitSubModuleDefinition def, PointsToContext context, ModuleObjectInternal value, ControlFlowNode origin) {
-        exists(PackageObjectInternal package |
-            package.getSourceModule() = def.getDefiningNode().getScope() |
-            value = package.submodule(def.getSourceVariable().getName()) and
-            origin = CfgOrigin::fromObject(value).asCfgNodeOrHere(def.getDefiningNode()) and
-            context.isImport()
-        )
+    predicate implicit_submodule_points_to(ImplicitSubModuleDefinition def, ModuleObjectInternal value, ControlFlowNode origin) {
+        value = getModule(def) and
+        origin = CfgOrigin::fromObject(value).asCfgNodeOrHere(def.getDefiningNode())
     }
 
     /** Points-to for `from ... import *`. */
@@ -1869,9 +1873,13 @@ cached module Types {
         result = getInheritedMetaclass(cls, 0)
         or
         // Best guess if base is not a known class
+        hasUnknownBase(cls) and result = ObjectInternal::unknownClass()
+    }
+
+    /* Helper for getInheritedMetaclass */
+    private predicate hasUnknownBase(ClassObjectInternal cls) {
         exists(ObjectInternal base |
-            base = getBase(cls, _) and
-            result = ObjectInternal::unknownClass() |
+            base = getBase(cls, _) |
             base.isClass() = false
             or
             base = ObjectInternal::unknownClass()
@@ -1881,14 +1889,18 @@ cached module Types {
     private ClassObjectInternal getInheritedMetaclass(ClassObjectInternal cls, int n) {
         exists(Class c |
             c = cls.(PythonClassObjectInternal).getScope() and
-            n = count(c.getABase())
+            n = count(c.getABase()) and n != 1
             |
             result = ObjectInternal::type() and major_version() = 3
             or
             result = ObjectInternal::classType() and major_version() = 2
         )
         or
+        base_count(cls) = 1 and n = 0 and
+        result = getBase(cls, 0).getClass()
+        or
         exists(ClassObjectInternal meta1, ClassObjectInternal meta2 |
+            base_count(cls) > 1 and
             meta1 = getBase(cls, n).getClass() and
             meta2 = getInheritedMetaclass(cls, n+1)
             |