Merge pull request #1632 from markshannon/python-account-for-dynamically-defined-builtin-instances

Python points-to: track more instances.

Author: psygnisfive

python/ql/src/Expressions/IncorrectComparisonUsingIs.ql

@@ -13,7 +13,7 @@
 import python
 import IsComparisons
 
-from Compare comp, Cmpop op, ClassObject c, string alt
+from Compare comp, Cmpop op, ClassValue c, string alt
 where invalid_portable_is_comparison(comp, op, c) and
 not cpython_interned_constant(comp.getASubExpression()) and
 (op instanceof Is and alt = "==" or op instanceof IsNot and alt = "!=")

python/ql/src/Expressions/IsComparisons.qll

@@ -7,25 +7,31 @@ predicate comparison_using_is(Compare comp, ControlFlowNode left, Cmpop op, Cont
     )
 }
 
-predicate overrides_eq_or_cmp(ClassObject c) {
+predicate overrides_eq_or_cmp(ClassValue c) {
     major_version() = 2 and c.hasAttribute("__eq__")
     or
-    c.declaresAttribute("__eq__") and not c = theObjectType()
+    c.declaresAttribute("__eq__") and not c = Value::named("object")
     or
-    exists(ClassObject sup | 
-        sup = c.getASuperType() and not sup = theObjectType() |
+    exists(ClassValue sup | 
+        sup = c.getASuperType() and not sup = Value::named("object") |
         sup.declaresAttribute("__eq__")
     )
     or
     major_version() = 2 and c.hasAttribute("__cmp__")
 }
 
-predicate invalid_to_use_is_portably(ClassObject c) {
+predicate probablySingleton(ClassValue cls) {
+    strictcount(Value inst | inst.getClass() = cls) = 1
+    or
+    cls = Value::named("None").getClass()
+}
+
+predicate invalid_to_use_is_portably(ClassValue c) {
     overrides_eq_or_cmp(c) and
     /* Exclude type/builtin-function/bool as it is legitimate to compare them using 'is' but they implement __eq__ */
-    not c = theTypeType() and not c = theBuiltinFunctionType() and not c = theBoolType() and
+    not c = Value::named("type") and not c = ClassValue::builtinFunction() and not c = Value::named("bool") and
     /* OK to compare with 'is' if a singleton */
-    not exists(c.getProbableSingletonInstance())
+    not probablySingleton(c)
 }
 
 predicate simple_constant(ControlFlowNode f) {
@@ -72,30 +78,30 @@ predicate universally_interned_constant(Expr e) {
     )
 }
 
-private predicate comparison_both_types(Compare comp, Cmpop op, ClassObject cls1, ClassObject cls2) {
+private predicate comparison_both_types(Compare comp, Cmpop op, ClassValue cls1, ClassValue cls2) {
     exists(ControlFlowNode op1, ControlFlowNode op2 |
         comparison_using_is(comp, op1, op, op2) or comparison_using_is(comp, op2, op, op1) |
-        op1.refersTo(_, cls1, _) and
-        op2.refersTo(_, cls2, _)
+        op1.inferredValue().getClass() = cls1 and
+        op2.inferredValue().getClass() = cls2
     )
 }
 
-private predicate comparison_one_type(Compare comp, Cmpop op, ClassObject cls) {
+private predicate comparison_one_type(Compare comp, Cmpop op, ClassValue cls) {
     not comparison_both_types(comp, _, _, _) and
     exists(ControlFlowNode operand |
         comparison_using_is(comp, operand, op, _) or comparison_using_is(comp, _, op, operand) |
-        operand.refersTo(_, cls, _)
+        operand.inferredValue().getClass() = cls 
     )
 }
 
-predicate invalid_portable_is_comparison(Compare comp, Cmpop op, ClassObject cls) {
+predicate invalid_portable_is_comparison(Compare comp, Cmpop op, ClassValue cls) {
     /* OK to use 'is' when defining '__eq__' */
     not exists(Function eq | eq.getName() = "__eq__" or eq.getName() = "__ne__" | eq = comp.getScope().getScope*())
     and
     (
         comparison_one_type(comp, op, cls) and invalid_to_use_is_portably(cls)
         or
-        exists(ClassObject other | comparison_both_types(comp, op, cls, other) |
+        exists(ClassValue other | comparison_both_types(comp, op, cls, other) |
             invalid_to_use_is_portably(cls) and
             invalid_to_use_is_portably(other)
         )
@@ -123,9 +129,9 @@ predicate invalid_portable_is_comparison(Compare comp, Cmpop op, ClassObject cls
 }
 
 private predicate enum_member(AstNode obj) {
-    exists(ClassObject cls, AssignStmt asgn |
+    exists(ClassValue cls, AssignStmt asgn |
         cls.getASuperType().getName() = "Enum" |
-        cls.getPyClass() = asgn.getScope() and
+        cls.getScope() = asgn.getScope() and
         asgn.getValue() = obj
     )
 }

python/ql/src/Expressions/NonPortableComparisonUsingIs.ql

@@ -13,7 +13,7 @@
 import python
 import IsComparisons
 
-from Compare comp, Cmpop op, ClassObject c
+from Compare comp, Cmpop op, ClassValue c
 where invalid_portable_is_comparison(comp, op, c) and
 exists(Expr sub | 
     sub = comp.getASubExpression() |

python/ql/src/semmle/python/objects/Constants.qll

@@ -78,6 +78,14 @@ abstract class ConstantObjectInternal extends ObjectInternal {
 
 }
 
+private boolean callToBool(CallNode call, PointsToContext context) {
+    PointsToInternal::pointsTo(call.getFunction(), context, ClassValue::bool(), _) and
+    exists(ObjectInternal arg |
+        PointsToInternal::pointsTo(call.getArg(0),context, arg, _) and
+        arg.booleanValue() = result
+    )
+}
+
 private abstract class BooleanObjectInternal extends ConstantObjectInternal {
 
     override ObjectInternal getClass() {
@@ -111,6 +119,8 @@ private class TrueObjectInternal extends BooleanObjectInternal, TTrue {
 
     override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
         node.(NameNode).getId() = "True" and context.appliesTo(node)
+        or
+        callToBool(node, context) = true
     }
 
     override int intValue() {
@@ -135,6 +145,8 @@ private class FalseObjectInternal extends BooleanObjectInternal, TFalse {
 
     override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
         node.(NameNode).getId() = "False" and context.appliesTo(node)
+        or
+        callToBool(node, context) = false
     }
 
     override int intValue() {

python/ql/src/semmle/python/objects/TObject.qll

@@ -469,7 +469,7 @@ library class ClassDecl extends @py_object {
         result = this.getClass().getName()
     }
 
-    /** Whether this is a class whose instances we treat specially, rather than as a generic instance.
+    /** Whether this is a class whose instances must be treated specially, rather than as generic instances.
      */
     predicate isSpecial() {
         exists(string name |
@@ -478,20 +478,13 @@ library class ClassDecl extends @py_object {
             name = "super" or
             name = "bool" or
             name = "NoneType" or
-            name = "int" or
-            name = "long" or
-            name = "str" or
-            name = "bytes" or
-            name = "unicode" or
             name = "tuple" or
             name = "property" or
             name = "ClassMethod" or
             name = "StaticMethod" or
             name = "MethodType" or
             name = "ModuleType"
         )
-        or
-        this = Builtin::builtin("float")
     }
 
     /** Holds if for class `C`, `C()` returns an instance of `C` */

python/ql/src/semmle/python/pointsto/PointsTo.qll

@@ -1249,7 +1249,13 @@ module Expressions {
             not op instanceof BitOr and
             (operand = left or operand = right) and
             PointsToInternal::pointsTo(operand, context, opvalue, _) and
-            value = ObjectInternal::unknown()
+            (
+                op instanceof Add and
+                value = TUnknownInstance(opvalue.getClass())
+                or
+                not op instanceof Add and
+                value = ObjectInternal::unknown()
+            )
             or
             op instanceof BitOr and
             exists(ObjectInternal lobj, ObjectInternal robj |

python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected

@@ -159,6 +159,7 @@
 | 106 | ControlFlowNode for inner | Function inner |
 | 108 | ControlFlowNode for IntegerLiteral | int 2 |
 | 108 | ControlFlowNode for z | int 2 |
+| 109 | ControlFlowNode for BinaryExpr | BinaryExpr |
 | 109 | ControlFlowNode for y | int 1 |
 | 109 | ControlFlowNode for z | int 2 |
 | 110 | ControlFlowNode for inner | Function inner |

python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected

@@ -159,6 +159,7 @@
 | 106 | ControlFlowNode for inner | Function inner | builtin-class function |
 | 108 | ControlFlowNode for IntegerLiteral | int 2 | builtin-class int |
 | 108 | ControlFlowNode for z | int 2 | builtin-class int |
+| 109 | ControlFlowNode for BinaryExpr | BinaryExpr | builtin-class int |
 | 109 | ControlFlowNode for y | int 1 | builtin-class int |
 | 109 | ControlFlowNode for z | int 2 | builtin-class int |
 | 110 | ControlFlowNode for inner | Function inner | builtin-class function |

python/ql/test/library-tests/PointsTo/new/PointsToUnknown.expected

@@ -167,12 +167,10 @@
 | c_tests.py:63 | ControlFlowNode for cond | 63 |
 | c_tests.py:73 | ControlFlowNode for x | 71 |
 | c_tests.py:73 | ControlFlowNode for y | 71 |
-| c_tests.py:74 | ControlFlowNode for BinaryExpr | 74 |
 | c_tests.py:74 | ControlFlowNode for x | 71 |
 | c_tests.py:74 | ControlFlowNode for y | 71 |
 | c_tests.py:76 | ControlFlowNode for x | 71 |
 | c_tests.py:76 | ControlFlowNode for y | 71 |
-| c_tests.py:77 | ControlFlowNode for BinaryExpr | 77 |
 | c_tests.py:77 | ControlFlowNode for x | 71 |
 | c_tests.py:77 | ControlFlowNode for y | 71 |
 | c_tests.py:80 | ControlFlowNode for IfExp | 80 |

python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected

@@ -416,6 +416,22 @@ WARNING: Predicate points_to has been deprecated and may be removed in future (P
 | h_classes.py:50 | ControlFlowNode for m | Function f | builtin-class function | 45 | import |
 | h_classes.py:52 | ControlFlowNode for FunctionExpr | Function n | builtin-class function | 52 | import |
 | h_classes.py:52 | ControlFlowNode for n | Function n | builtin-class function | 52 | import |
+| h_classes.py:55 | ControlFlowNode for int | builtin-class int | builtin-class type | 55 | import |
+| h_classes.py:55 | ControlFlowNode for int() | int() | builtin-class int | 55 | import |
+| h_classes.py:56 | ControlFlowNode for Str | '' | builtin-class str | 56 | import |
+| h_classes.py:56 | ControlFlowNode for type | builtin-class type | builtin-class type | 56 | import |
+| h_classes.py:56 | ControlFlowNode for type() | builtin-class str | builtin-class type | 56 | import |
+| h_classes.py:56 | ControlFlowNode for type()() | type()() | builtin-class str | 56 | import |
+| h_classes.py:57 | ControlFlowNode for list | builtin-class list | builtin-class type | 57 | import |
+| h_classes.py:57 | ControlFlowNode for list() | list() | builtin-class list | 57 | import |
+| h_classes.py:58 | ControlFlowNode for dict | builtin-class dict | builtin-class type | 58 | import |
+| h_classes.py:58 | ControlFlowNode for dict() | dict() | builtin-class dict | 58 | import |
+| h_classes.py:59 | ControlFlowNode for Str | 'hi' | builtin-class str | 59 | import |
+| h_classes.py:59 | ControlFlowNode for bool | builtin-class bool | builtin-class type | 59 | import |
+| h_classes.py:59 | ControlFlowNode for bool() | bool True | builtin-class bool | 59 | import |
+| h_classes.py:60 | ControlFlowNode for IntegerLiteral | int 0 | builtin-class int | 60 | import |
+| h_classes.py:60 | ControlFlowNode for bool | builtin-class bool | builtin-class type | 60 | import |
+| h_classes.py:60 | ControlFlowNode for bool() | bool False | builtin-class bool | 60 | import |
 | i_imports.py:3 | ControlFlowNode for IntegerLiteral | int 1 | builtin-class int | 3 | import |
 | i_imports.py:3 | ControlFlowNode for a | int 1 | builtin-class int | 3 | import |
 | i_imports.py:4 | ControlFlowNode for IntegerLiteral | int 2 | builtin-class int | 4 | import |