Merge pull request #1391 from markshannon/python-points-to-varargs

taus-semmle · web-flow · commit 58285c08dd3d · 2019-06-05T20:17:01.000+02:00
Python points to varargs
diff --git a/python/ql/src/semmle/python/Exprs.qll b/python/ql/src/semmle/python/Exprs.qll
@@ -168,12 +168,12 @@ class Call extends Call_ {
 
     override CallNode getAFlowNode() { result = super.getAFlowNode() }
 
-    /** Gets a tuple (*) argument of this class definition. */
+    /** Gets a tuple (*) argument of this call. */
     Expr getStarargs() {
         result = this.getAPositionalArg().(Starred).getValue()
     }
 
-    /** Gets a dictionary (**) argument of this class definition. */
+    /** Gets a dictionary (**) argument of this call. */
     Expr getKwargs() {
         result = this.getANamedArg().(DictUnpacking).getValue()
     }
@@ -227,6 +227,18 @@ class Call extends Call_ {
         result = this.getKwargs().(Dict).getAKey().(StrConst).getText()
     }
 
+    /** Gets the positional argument count of this call, provided there is no more than one tuple (*) argument. */
+    int getPositionalArgumentCount() {
+        count(this.getStarargs()) < 2 and
+        result = count(Expr arg | arg = this.getAPositionalArg() and not arg instanceof Starred)
+    }
+
+    /** Gets the tuple (*) argument of this call, provided there is exactly one. */
+    Expr getStarArg() {
+        count(this.getStarargs()) < 2 and
+        result = getStarargs()
+    }
+
 }
 
 /** A conditional expression such as, `body if test else orelse` */
diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll
@@ -481,6 +481,12 @@ class CallNode extends ControlFlowNode {
         )
     }
 
+    /** Gets the tuple (*) argument of this call, provided there is exactly one. */
+    ControlFlowNode getStarArg() {
+        result.getNode() = this.getNode().getStarArg() and
+        result.getBasicBlock().dominates(this.getBasicBlock())
+    }
+
 }
 
 /** A control flow corresponding to an attribute expression, such as `value.attr` */
diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll
@@ -481,6 +481,10 @@ module ObjectInternal {
         result = TBuiltinClassObject(Builtin::special("ClassType"))
     }
 
+    ObjectInternal emptyTuple() {
+        result.(BuiltinTupleObjectInternal).length() = 0
+    }
+
 }
 
 /** Helper for boolean predicates returning both `true` and `false` */
diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll
@@ -39,7 +39,9 @@ abstract class TupleObjectInternal extends SequenceObjectInternal {
     }
 
     private string contents(int n) {
-        n = this.length() and result = ""
+        n < 4 and n = this.length() and result = ""
+        or
+        n = 3 and this.length() > 3 and result = (this.length()-3).toString() + " more..."
         or
         result = this.getItem(n).toString() + ", " + this.contents(n+1)
     }
@@ -88,6 +90,7 @@ abstract class TupleObjectInternal extends SequenceObjectInternal {
 
 }
 
+/** A tuple built-in to the interpreter, including the empty tuple. */
 class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal {
 
     override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
@@ -114,7 +117,7 @@ class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal {
     }
 }
 
-
+/** A tuple declared by a tuple expression in the Python source code */
 class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal {
 
     override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
@@ -145,6 +148,35 @@ class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal {
 
 }
 
+/** A tuple created by a `*` parameter */
+class VarargsTupleObjectInternal extends TVarargsTuple,  TupleObjectInternal {
+
+    override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
+        none()
+    }
+
+    override Builtin getBuiltin() {
+        none()
+    }
+
+    override ControlFlowNode getOrigin() {
+        none()
+    }
+
+    override ObjectInternal getItem(int n) {
+        exists(CallNode call, PointsToContext context, int offset, int length |
+            this = TVarargsTuple(call, context, offset, length) and
+            n < length and
+            InterProceduralPointsTo::positional_argument_points_to(call, offset+n, context, result, _)
+        )
+    }
+
+    override int length() {
+        this = TVarargsTuple(_, _, _, result)
+    }
+}
+
+
 /** The `sys.version_info` object. We treat this specially to prevent premature pruning and
  * false positives when we are unsure of the actual version of Python that the code is expecting.
  */
diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll
@@ -179,6 +179,11 @@ cached newtype TObject =
         context.appliesTo(origin)
     }
     or
+    /* Varargs tuple */
+    TVarargsTuple(CallNode call, PointsToContext context, int offset, int length) {
+        InterProceduralPointsTo::varargs_tuple(call, context, _, _, offset, length)
+    }
+    or
     /* `type` */
     TType()
     or
diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll
@@ -844,11 +844,13 @@ module InterProceduralPointsTo {
     private predicate normal_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) {
         exists(PointsToContext caller, ControlFlowNode arg |
             PointsToInternal::pointsTo(arg, caller, value, origin) and
-            callsite_argument_transfer(arg, caller, def, context)
+            named_argument_transfer(arg, caller, def, context)
         )
         or
         not def.isSelf() and not def.isVarargs() and not def.isKwargs() and
         context.isRuntime() and value = ObjectInternal::unknown() and origin = def.getDefiningNode()
+        or
+        positional_parameter_points_to(def, context, value, origin)
     }
 
     pragma [noinline]
@@ -898,44 +900,107 @@ module InterProceduralPointsTo {
     /** Helper for parameter_points_to */
     pragma [noinline]
     private predicate special_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) {
+        /* Runtime: Just an unknown tuple (or dict for `**` args) */
         special_parameter_value(def, value) and
-        (
-            context.isRuntime()
-            or
-            exists(PointsToContext caller, CallNode call |
-                context.fromCall(call, caller) and
-                context.appliesToScope(def.getScope()) and
-                not exists(call.getArg(def.getParameter().getPosition())) and
-                not exists(call.getArgByName(def.getParameter().getName()))
-            )
+        context.isRuntime() and
+        origin = def.getDefiningNode()
+        or
+        /* A tuple constructed from positional arguments for a `*` parameter. */
+        def.isVarargs() and
+        exists(CallNode call, Function scope, PointsToContext caller, int offset, int length |
+            varargs_tuple(call, caller, scope, context, offset, length) and
+            value = TVarargsTuple(call, caller, offset, length) and
+            def.getScope() = scope
+        ) and
+        origin = def.getDefiningNode()
+        or
+        /* A `*` parameter with no surplus positional arguments; an empty tuple */
+        def.isVarargs() and
+        exists(Function scope |
+            varargs_empty_tuple(scope, context) and
+            value = ObjectInternal::emptyTuple() and
+            def.getScope() = scope
         ) and
         origin = def.getDefiningNode()
     }
 
+    /** Holds if `call` in context `caller` calls into the function scope `scope` in context `callee` and
+     * that the number of position arguments (including expansion of `*` argument) exceeds the number of positional arguments by
+     * `length` and that the excess arguments start at `start`.
+     */
+    predicate varargs_tuple(CallNode call, PointsToContext caller, Function scope, PointsToContext callee, int start, int length) {
+        exists(int parameter_offset |
+            callsite_calls_function(call, caller, scope, callee, parameter_offset) and
+            start = scope.getPositionalParameterCount() - parameter_offset and
+            length = positional_argument_count(call, caller) - start and
+            length > 0
+        )
+    }
+
+    /** Holds if for function scope `func` in context `callee` the `*` parameter will hold the empty tuple. */
+    predicate varargs_empty_tuple(Function func, PointsToContext callee) {
+        exists(CallNode call, PointsToContext caller, int parameter_offset |
+            callsite_calls_function(call, caller, func, callee, parameter_offset) and
+            func.getPositionalParameterCount() - parameter_offset >= positional_argument_count(call, caller)
+        )
+    }
+
     /** Helper predicate for special_parameter_points_to */
     private predicate special_parameter_value(ParameterDefinition p, ObjectInternal value) {
         p.isVarargs() and value = TUnknownInstance(ObjectInternal::builtin("tuple"))
         or
         p.isKwargs() and value = TUnknownInstance(ObjectInternal::builtin("dict"))
     }
 
-    /** Holds if the `(argument, caller)` pair matches up with `(param, callee)` pair across call. */
-    cached predicate callsite_argument_transfer(ControlFlowNode argument, PointsToContext caller, ParameterDefinition param, PointsToContext callee) {
+    /** Holds if the `n`th argument in call `call` with context `caller` points-to `value` from `origin`, including values in tuples
+     * expanded by a `*` argument. For example, for the call `f('a', *(`x`,`y`))` the arguments are `('a', 'x', y')`
+     */
+    predicate positional_argument_points_to(CallNode call, int n, PointsToContext caller, ObjectInternal value, ControlFlowNode origin) {
+        PointsToInternal::pointsTo(call.getArg(n), caller, value, origin)
+        or
+        exists(SequenceObjectInternal arg, int pos |
+            pos = call.getNode().getPositionalArgumentCount() and
+            PointsToInternal::pointsTo(origin, caller, arg, _) and
+            value = arg.getItem(n-pos) and
+            origin = call.getStarArg()
+        )
+    }
+
+    /** Gets the number of positional arguments including values in tuples expanded by a `*` argument.*/
+    private int positional_argument_count(CallNode call, PointsToContext caller) {
+        result = call.getNode().getPositionalArgumentCount() and not exists(call.getStarArg()) and caller.appliesTo(call)
+        or
+        exists(SequenceObjectInternal arg, int pos |
+            pos = call.getNode().getPositionalArgumentCount() and
+            PointsToInternal::pointsTo(call.getStarArg(), caller, arg, _) and
+            result = pos + arg.length()
+        )
+    }
+
+    /** Holds if the parameter definition `def` points-to `value` from `origin` given the context `context` */
+    predicate positional_parameter_points_to(ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin) {
+        exists(CallNode call, int argument, PointsToContext caller, Function func, int offset |
+            positional_argument_points_to(call, argument, caller, value, origin) and
+            callsite_calls_function(call, caller, func, context, offset) and
+            def.getParameter() = func.getArg(argument+offset)
+        )
+    }
+
+    /** Holds if the named `argument` given the context `caller` is transferred to the parameter `param` with conntext `callee` by a call. */
+    cached predicate named_argument_transfer(ControlFlowNode argument, PointsToContext caller, ParameterDefinition param, PointsToContext callee) {
         exists(CallNode call, Function func, int offset |
             callsite_calls_function(call, caller, func, callee, offset)
             |
-            exists(int n |
-                argument = call.getArg(n) and
-                param.getParameter() = func.getArg(n+offset)
-            )
-            or
             exists(string name |
                 argument = call.getArgByName(name) and
                 param.getParameter() = func.getArgByName(name)
             )
         )
     }
 
+    /** Holds if the `call` with context `caller` calls the function `scope` in context `callee` 
+     * and the offset from argument to parameter is `parameter_offset`
+     */
     cached predicate callsite_calls_function(CallNode call, PointsToContext caller, Function scope, PointsToContext callee, int parameter_offset) {
         exists(ObjectInternal func |
             callWithContext(call, caller, func, callee) and
diff --git a/python/ql/test/library-tests/PointsTo/new/Call.expected b/python/ql/test/library-tests/PointsTo/new/Call.expected
@@ -16,6 +16,13 @@
 | l_calls.py:10 | ControlFlowNode for bar() | bar |
 | l_calls.py:24 | ControlFlowNode for Attribute() | Owner.cm |
 | l_calls.py:25 | ControlFlowNode for Attribute() | Owner.cm2 |
+| l_calls.py:37 | ControlFlowNode for f() | f |
+| l_calls.py:38 | ControlFlowNode for Attribute() | E.m |
+| l_calls.py:39 | ControlFlowNode for Attribute() | E.m |
+| l_calls.py:42 | ControlFlowNode for f() | f |
+| l_calls.py:51 | ControlFlowNode for g() | g |
+| l_calls.py:52 | ControlFlowNode for Attribute() | F.m |
+| l_calls.py:53 | ControlFlowNode for Attribute() | F.m |
 | q_super.py:4 | ControlFlowNode for Attribute() | object.__init__ |
 | q_super.py:12 | ControlFlowNode for Attribute() | Base2.__init__ |
 | q_super.py:22 | ControlFlowNode for Attribute() | Base1.meth |
diff --git a/python/ql/test/library-tests/PointsTo/new/NameSpace.expected b/python/ql/test/library-tests/PointsTo/new/NameSpace.expected
@@ -112,12 +112,19 @@
 | k_getsetattr.py:0 | Module code.k_getsetattr | k | Function k |
 | k_getsetattr.py:4 | Class C | meth1 | Function meth1 |
 | k_getsetattr.py:4 | Class C | meth2 | Function meth2 |
+| l_calls.py:0 | Module code.l_calls | E | class E |
+| l_calls.py:0 | Module code.l_calls | F | class F |
 | l_calls.py:0 | Module code.l_calls | Owner | class Owner |
 | l_calls.py:0 | Module code.l_calls | bar | Function bar |
+| l_calls.py:0 | Module code.l_calls | f | Function f |
 | l_calls.py:0 | Module code.l_calls | foo | Function foo |
+| l_calls.py:0 | Module code.l_calls | g | Function g |
+| l_calls.py:0 | Module code.l_calls | t | Tuple |
 | l_calls.py:12 | Class Owner | cm | classmethod() |
 | l_calls.py:12 | Class Owner | cm2 | classmethod() |
 | l_calls.py:12 | Class Owner | m | Function m |
+| l_calls.py:32 | Class E | m | Function m |
+| l_calls.py:47 | Class F | m | Function m |
 | o_no_returns.py:0 | Module code.o_no_returns | bar | Function bar |
 | o_no_returns.py:0 | Module code.o_no_returns | fail | Function fail |
 | o_no_returns.py:0 | Module code.o_no_returns | foo | Function foo |
diff --git a/python/ql/test/library-tests/PointsTo/new/PointsToNone.expected b/python/ql/test/library-tests/PointsTo/new/PointsToNone.expected
@@ -75,6 +75,7 @@
 | k_getsetattr.py:15 | ControlFlowNode for Attribute() | 6 |
 | l_calls.py:4 | ControlFlowNode for Attribute() | 4 |
 | l_calls.py:9 | ControlFlowNode for foo() | 4 |
+| l_calls.py:48 | ControlFlowNode for None | 48 |
 | m_attributes.py:12 | ControlFlowNode for Attribute() | 8 |
 | m_attributes.py:13 | ControlFlowNode for Attribute() | 8 |
 | o_no_returns.py:7 | ControlFlowNode for fail() | 10 |
diff --git a/python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected b/python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected
diff --git a/python/ql/test/library-tests/PointsTo/new/PointsToWithType.expected b/python/ql/test/library-tests/PointsTo/new/PointsToWithType.expected
diff --git a/python/ql/test/library-tests/PointsTo/new/Values.expected b/python/ql/test/library-tests/PointsTo/new/Values.expected
diff --git a/python/ql/test/library-tests/PointsTo/new/code/l_calls.py b/python/ql/test/library-tests/PointsTo/new/code/l_calls.py

Original file line number	Diff line number	Diff line change
`@@ -481,6 +481,12 @@ class CallNode extends ControlFlowNode {`
`481`	`481`	`)`
`482`	`482`	`}`
`483`	`483`
	`484`	`+ /** Gets the tuple () argument of this call, provided there is exactly one. /`
	`485`	`+ ControlFlowNode getStarArg() {`
	`486`	`+ result.getNode() = this.getNode().getStarArg() and`
	`487`	`+ result.getBasicBlock().dominates(this.getBasicBlock())`
	`488`	`+ }`
	`489`	`+`
`484`	`490`	`}`
`485`	`491`
`486`	`492`	/** A control flow corresponding to an attribute expression, such as `value.attr` */
Original file line number	Diff line number	Diff line change
`@@ -481,6 +481,10 @@ module ObjectInternal {`
`481`	`481`	`result = TBuiltinClassObject(Builtin::special("ClassType"))`
`482`	`482`	`}`
`483`	`483`
	`484`	`+ ObjectInternal emptyTuple() {`
	`485`	`+ result.(BuiltinTupleObjectInternal).length() = 0`
	`486`	`+ }`
	`487`	`+`
`484`	`488`	`}`
`485`	`489`
`486`	`490`	/** Helper for boolean predicates returning both `true` and `false` */