Skip to content

Commit 58baec5

Browse files
RasmusWLRasmusWL
authored
Merge pull request #4364 from yoff/SharedDataflow_ArgumentPassing
Python: Shared dataflow, argument passing
2 parents 388f60f + 9c8e968 commit 58baec5

24 files changed

+2703
-1551
lines changed

‎python/ql/src/experimental/dataflow/internal/DataFlowPrivate.qll‎

Lines changed: 378 additions & 20 deletions
Large diffs are not rendered by default.

‎python/ql/src/experimental/dataflow/internal/DataFlowPublic.qll‎

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,40 @@ newtype TNode =
2626
TCfgNode(ControlFlowNode node) { isExpressionNode(node) } or
2727
/** A synthetic node representing the value of an object before a state change */
2828
TSyntheticPreUpdateNode(NeedsSyntheticPreUpdateNode post) or
29-
/** A synthetic node representing the value of an object after a state change */
29+
/** A synthetic node representing the value of an object after a state change. */
3030
TSyntheticPostUpdateNode(NeedsSyntheticPostUpdateNode pre) or
31-
/** A node representing a global (module-level) variable in a specific module */
32-
TModuleVariableNode(Module m, GlobalVariable v) { v.getScope() = m and v.escapes() }
31+
/** A node representing a global (module-level) variable in a specific module. */
32+
TModuleVariableNode(Module m, GlobalVariable v) { v.getScope() = m and v.escapes() } or
33+
/**
34+
* A node representing the overflow positional arguments to a call.
35+
* That is, `call` contains more positional arguments than there are
36+
* positional parameters in `callable`. The extra ones are passed as
37+
* a tuple to a starred parameter; this synthetic node represents that tuple.
38+
*/
39+
TPosOverflowNode(CallNode call, CallableValue callable) {
40+
exists(getPositionalOverflowArg(call, callable, _))
41+
} or
42+
/**
43+
* A node representing the overflow keyword arguments to a call.
44+
* That is, `call` contains keyword arguments for keys that do not have
45+
* keyword parameters in `callable`. These extra ones are passed as
46+
* a dictionary to a doubly starred parameter; this synthetic node
47+
* represents that dictionary.
48+
*/
49+
TKwOverflowNode(CallNode call, CallableValue callable) {
50+
exists(getKeywordOverflowArg(call, callable, _))
51+
or
52+
exists(call.getNode().getKwargs()) and
53+
callable.getScope().hasKwArg()
54+
} or
55+
/**
56+
* A node representing an unpacked element of a dictionary argument.
57+
* That is, `call` contains argument `**{"foo": bar}` which is passed
58+
* to parameter `foo` of `callable`.
59+
*/
60+
TKwUnpacked(CallNode call, CallableValue callable, string name) {
61+
call_unpacks(call, _, callable, name, _)
62+
}
3363

3464
/**
3565
* An element, viewed as a node in a data flow graph. Either an SSA variable
@@ -240,6 +270,49 @@ class ModuleVariableNode extends Node, TModuleVariableNode {
240270
override Location getLocation() { result = mod.getLocation() }
241271
}
242272

273+
/**
274+
* The node holding the extra positional arguments to a call. This node is passed as a tuple
275+
* to the starred parameter of the callable.
276+
*/
277+
class PosOverflowNode extends Node, TPosOverflowNode {
278+
CallNode call;
279+
280+
PosOverflowNode() { this = TPosOverflowNode(call, _) }
281+
282+
override string toString() { result = "PosOverflowNode for " + call.getNode().toString() }
283+
284+
override Location getLocation() { result = call.getLocation() }
285+
}
286+
287+
/**
288+
* The node holding the extra keyword arguments to a call. This node is passed as a dictionary
289+
* to the doubly starred parameter of the callable.
290+
*/
291+
class KwOverflowNode extends Node, TKwOverflowNode {
292+
CallNode call;
293+
294+
KwOverflowNode() { this = TKwOverflowNode(call, _) }
295+
296+
override string toString() { result = "KwOverflowNode for " + call.getNode().toString() }
297+
298+
override Location getLocation() { result = call.getLocation() }
299+
}
300+
301+
/**
302+
* The node representing the synthetic argument of a call that is unpacked from a dictionary
303+
* argument.
304+
*/
305+
class KwUnpacked extends Node, TKwUnpacked {
306+
CallNode call;
307+
string name;
308+
309+
KwUnpacked() { this = TKwUnpacked(call, _, name) }
310+
311+
override string toString() { result = "KwUnpacked " + name }
312+
313+
override Location getLocation() { result = call.getLocation() }
314+
}
315+
243316
/**
244317
* A node that controls whether other nodes are evaluated.
245318
*/

‎python/ql/test/experimental/dataflow/coverage/argumentPassing.py‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
import sys
2+
import os
3+
4+
sys.path.append(os.path.dirname(os.path.dirname((__file__))))
5+
from testlib import *
6+
7+
arg = "source"
8+
arg1 = "source1"
9+
arg2 = "source2"
10+
arg3 = "source3"
11+
arg4 = "source4"
12+
arg5 = "source5"
13+
arg6 = "source6"
14+
arg7 = "source7"
15+
16+
17+
def SINK_TEST(x, test):
18+
if test(x):
19+
print("OK")
20+
else:
21+
print("Unexpected flow", x)
22+
23+
24+
def SINK(x, expected=arg):
25+
SINK_TEST(x, test=lambda x: x == expected)
26+
27+
28+
def SINK_F(x, unexpected=arg):
29+
SINK_TEST(x, test=lambda x: x != unexpected)
30+
31+
32+
def SINK1(x):
33+
SINK(x, expected=arg1)
34+
35+
36+
def SINK2(x):
37+
SINK(x, expected=arg2)
38+
39+
40+
def SINK2_F(x):
41+
SINK_F(x, unexpected=arg2)
42+
43+
44+
def SINK3(x):
45+
SINK(x, expected=arg3)
46+
47+
48+
def SINK4(x):
49+
SINK(x, expected=arg4)
50+
51+
52+
def SINK5(x):
53+
SINK(x, expected=arg5)
54+
55+
56+
def SINK6(x):
57+
SINK(x, expected=arg6)
58+
59+
60+
def SINK7(x):
61+
SINK(x, expected=arg7)
62+
63+
64+
def argument_passing(
65+
a,
66+
b,
67+
/,
68+
c,
69+
d=arg4,
70+
*,
71+
e=arg5,
72+
f,
73+
**g,
74+
):
75+
SINK1(a)
76+
SINK2(b)
77+
SINK3(c)
78+
SINK4(d)
79+
SINK5(e)
80+
SINK6(f)
81+
try:
82+
SINK7(g["g"])
83+
except:
84+
print("OK")
85+
86+
87+
@expects(7)
88+
def test_argument_passing1():
89+
argument_passing(arg1, *(arg2, arg3, arg4), e=arg5, **{"f": arg6, "g": arg7})
90+
91+
92+
@expects(7)
93+
def test_argument_passing2():
94+
argument_passing(arg1, arg2, arg3, f=arg6)
95+
96+
97+
def with_pos_only(a, /, b):
98+
SINK1(a)
99+
SINK2(b)
100+
101+
102+
@expects(6)
103+
def test_pos_only():
104+
with_pos_only(arg1, arg2)
105+
with_pos_only(arg1, b=arg2)
106+
with_pos_only(arg1, *(arg2,))
107+
108+
109+
def with_multiple_kw_args(a, b, c):
110+
SINK1(a)
111+
SINK2(b)
112+
SINK3(c)
113+
114+
115+
@expects(9)
116+
def test_multiple_kw_args():
117+
with_multiple_kw_args(b=arg2, c=arg3, a=arg1)
118+
with_multiple_kw_args(arg1, *(arg2,), arg3)
119+
with_multiple_kw_args(arg1, **{"c": arg3}, b=arg2)
120+
with_multiple_kw_args(**{"b": arg2}, **{"c": arg3}, **{"a": arg1})
121+
122+
123+
def with_default_arguments(a=arg1, b=arg2, c=arg3):
124+
SINK1(a)
125+
SINK2(b)
126+
SINK3(c)
127+
128+
129+
@expects(12)
130+
def test_default_arguments():
131+
with_default_arguments()
132+
with_default_arguments(arg1)
133+
with_default_arguments(b=arg2)
134+
with_default_arguments(**{"c": arg3})
135+
136+
137+
# Nested constructor pattern
138+
def grab_foo_bar_baz(foo, **kwargs):
139+
SINK1(foo)
140+
grab_bar_baz(**kwargs)
141+
142+
143+
# It is not possible to pass `bar` into `kwargs`,
144+
# since `bar` is a valid keyword argument.
145+
def grab_bar_baz(bar, **kwargs):
146+
SINK2(bar)
147+
try:
148+
SINK2_F(kwargs["bar"])
149+
except:
150+
print("OK")
151+
grab_baz(**kwargs)
152+
153+
154+
def grab_baz(baz):
155+
SINK3(baz)
156+
157+
158+
@expects(4)
159+
def test_grab():
160+
grab_foo_bar_baz(baz=arg3, bar=arg2, foo=arg1)
161+
162+
163+
# All combinations
164+
def test_pos_pos():
165+
def with_pos(a):
166+
SINK1(a)
167+
168+
with_pos(arg1)
169+
170+
171+
def test_pos_pos_only():
172+
def with_pos_only(a, /):
173+
SINK1(a)
174+
175+
with_pos_only(arg1)
176+
177+
178+
def test_pos_star():
179+
def with_star(*a):
180+
if len(a) > 0:
181+
SINK1(a[0])
182+
183+
with_star(arg1)
184+
185+
186+
def test_pos_kw():
187+
def with_kw(a=""):
188+
SINK1(a)
189+
190+
with_kw(arg1)
191+
192+
193+
def test_kw_pos():
194+
def with_pos(a):
195+
SINK1(a)
196+
197+
with_pos(a=arg1)
198+
199+
200+
def test_kw_kw():
201+
def with_kw(a=""):
202+
SINK1(a)
203+
204+
with_kw(a=arg1)
205+
206+
207+
def test_kw_doublestar():
208+
def with_doublestar(**a):
209+
SINK1(a["a"])
210+
211+
with_doublestar(a=arg1)

‎python/ql/test/experimental/dataflow/coverage/argumentRouting1.expected‎

Lines changed: 34 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,34 @@
1-
| classes.py:620:5:620:16 | SSA variable with_getitem | classes.py:614:15:614:18 | ControlFlowNode for self |
2-
| classes.py:637:5:637:16 | SSA variable with_setitem | classes.py:632:15:632:18 | ControlFlowNode for self |
3-
| classes.py:654:5:654:16 | SSA variable with_delitem | classes.py:649:15:649:18 | ControlFlowNode for self |
4-
| classes.py:735:5:735:12 | SSA variable with_add | classes.py:729:15:729:18 | ControlFlowNode for self |
5-
| classes.py:752:5:752:12 | SSA variable with_sub | classes.py:746:15:746:18 | ControlFlowNode for self |
6-
| classes.py:769:5:769:12 | SSA variable with_mul | classes.py:763:15:763:18 | ControlFlowNode for self |
7-
| classes.py:786:5:786:15 | SSA variable with_matmul | classes.py:780:15:780:18 | ControlFlowNode for self |
8-
| classes.py:803:5:803:16 | SSA variable with_truediv | classes.py:797:15:797:18 | ControlFlowNode for self |
9-
| classes.py:820:5:820:17 | SSA variable with_floordiv | classes.py:814:15:814:18 | ControlFlowNode for self |
10-
| classes.py:837:5:837:12 | SSA variable with_mod | classes.py:831:15:831:18 | ControlFlowNode for self |
11-
| classes.py:877:5:877:12 | SSA variable with_pow | classes.py:865:15:865:18 | ControlFlowNode for self |
12-
| classes.py:894:5:894:15 | SSA variable with_lshift | classes.py:888:15:888:18 | ControlFlowNode for self |
13-
| classes.py:911:5:911:15 | SSA variable with_rshift | classes.py:905:15:905:18 | ControlFlowNode for self |
14-
| classes.py:928:5:928:12 | SSA variable with_and | classes.py:922:15:922:18 | ControlFlowNode for self |
15-
| classes.py:945:5:945:12 | SSA variable with_xor | classes.py:939:15:939:18 | ControlFlowNode for self |
16-
| classes.py:962:5:962:11 | SSA variable with_or | classes.py:956:15:956:18 | ControlFlowNode for self |
1+
| argumentPassing.py:89:22:89:25 | ControlFlowNode for arg1 | argumentPassing.py:75:11:75:11 | ControlFlowNode for a |
2+
| argumentPassing.py:94:22:94:25 | ControlFlowNode for arg1 | argumentPassing.py:75:11:75:11 | ControlFlowNode for a |
3+
| argumentPassing.py:104:19:104:22 | ControlFlowNode for arg1 | argumentPassing.py:98:11:98:11 | ControlFlowNode for a |
4+
| argumentPassing.py:105:19:105:22 | ControlFlowNode for arg1 | argumentPassing.py:98:11:98:11 | ControlFlowNode for a |
5+
| argumentPassing.py:106:19:106:22 | ControlFlowNode for arg1 | argumentPassing.py:98:11:98:11 | ControlFlowNode for a |
6+
| argumentPassing.py:117:45:117:48 | ControlFlowNode for arg1 | argumentPassing.py:110:11:110:11 | ControlFlowNode for a |
7+
| argumentPassing.py:118:27:118:30 | ControlFlowNode for arg1 | argumentPassing.py:110:11:110:11 | ControlFlowNode for a |
8+
| argumentPassing.py:119:27:119:30 | ControlFlowNode for arg1 | argumentPassing.py:110:11:110:11 | ControlFlowNode for a |
9+
| argumentPassing.py:120:65:120:68 | ControlFlowNode for arg1 | argumentPassing.py:110:11:110:11 | ControlFlowNode for a |
10+
| argumentPassing.py:132:28:132:31 | ControlFlowNode for arg1 | argumentPassing.py:124:11:124:11 | ControlFlowNode for a |
11+
| argumentPassing.py:160:46:160:49 | ControlFlowNode for arg1 | argumentPassing.py:139:11:139:13 | ControlFlowNode for foo |
12+
| argumentPassing.py:168:14:168:17 | ControlFlowNode for arg1 | argumentPassing.py:166:15:166:15 | ControlFlowNode for a |
13+
| argumentPassing.py:175:19:175:22 | ControlFlowNode for arg1 | argumentPassing.py:173:15:173:15 | ControlFlowNode for a |
14+
| argumentPassing.py:183:15:183:18 | ControlFlowNode for arg1 | argumentPassing.py:181:19:181:22 | ControlFlowNode for Subscript |
15+
| argumentPassing.py:190:13:190:16 | ControlFlowNode for arg1 | argumentPassing.py:188:15:188:15 | ControlFlowNode for a |
16+
| argumentPassing.py:197:16:197:19 | ControlFlowNode for arg1 | argumentPassing.py:195:15:195:15 | ControlFlowNode for a |
17+
| argumentPassing.py:204:15:204:18 | ControlFlowNode for arg1 | argumentPassing.py:202:15:202:15 | ControlFlowNode for a |
18+
| argumentPassing.py:211:23:211:26 | ControlFlowNode for arg1 | argumentPassing.py:209:15:209:20 | ControlFlowNode for Subscript |
19+
| classes.py:563:5:563:16 | SSA variable with_getitem | classes.py:557:15:557:18 | ControlFlowNode for self |
20+
| classes.py:578:5:578:16 | SSA variable with_setitem | classes.py:573:15:573:18 | ControlFlowNode for self |
21+
| classes.py:593:5:593:16 | SSA variable with_delitem | classes.py:588:15:588:18 | ControlFlowNode for self |
22+
| classes.py:665:5:665:12 | SSA variable with_add | classes.py:659:15:659:18 | ControlFlowNode for self |
23+
| classes.py:680:5:680:12 | SSA variable with_sub | classes.py:674:15:674:18 | ControlFlowNode for self |
24+
| classes.py:695:5:695:12 | SSA variable with_mul | classes.py:689:15:689:18 | ControlFlowNode for self |
25+
| classes.py:710:5:710:15 | SSA variable with_matmul | classes.py:704:15:704:18 | ControlFlowNode for self |
26+
| classes.py:725:5:725:16 | SSA variable with_truediv | classes.py:719:15:719:18 | ControlFlowNode for self |
27+
| classes.py:740:5:740:17 | SSA variable with_floordiv | classes.py:734:15:734:18 | ControlFlowNode for self |
28+
| classes.py:755:5:755:12 | SSA variable with_mod | classes.py:749:15:749:18 | ControlFlowNode for self |
29+
| classes.py:791:5:791:12 | SSA variable with_pow | classes.py:779:15:779:18 | ControlFlowNode for self |
30+
| classes.py:806:5:806:15 | SSA variable with_lshift | classes.py:800:15:800:18 | ControlFlowNode for self |
31+
| classes.py:821:5:821:15 | SSA variable with_rshift | classes.py:815:15:815:18 | ControlFlowNode for self |
32+
| classes.py:836:5:836:12 | SSA variable with_and | classes.py:830:15:830:18 | ControlFlowNode for self |
33+
| classes.py:851:5:851:12 | SSA variable with_xor | classes.py:845:15:845:18 | ControlFlowNode for self |
34+
| classes.py:866:5:866:11 | SSA variable with_or | classes.py:860:15:860:18 | ControlFlowNode for self |

‎python/ql/test/experimental/dataflow/coverage/argumentRouting1.ql‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ class ArgumentRoutingConfig extends DataFlow::Configuration {
99
ArgumentRoutingConfig() { this = "ArgumentRoutingConfig" }
1010

1111
override predicate isSource(DataFlow::Node node) {
12+
node.(DataFlow::CfgNode).getNode().(NameNode).getId() = "arg1"
13+
or
1214
exists(AssignmentDefinition def, DataFlowPrivate::DataFlowCall call |
1315
def.getVariable() = node.(DataFlow::EssaNode).getVar() and
1416
def.getValue() = call.getNode() and
@@ -27,7 +29,7 @@ class ArgumentRoutingConfig extends DataFlow::Configuration {
2729

2830
from DataFlow::Node source, DataFlow::Node sink
2931
where
30-
source.getLocation().getFile().getBaseName() = "classes.py" and
31-
sink.getLocation().getFile().getBaseName() = "classes.py" and
32+
source.getLocation().getFile().getBaseName() in ["classes.py", "argumentPassing.py"] and
33+
sink.getLocation().getFile().getBaseName() in ["classes.py", "argumentPassing.py"] and
3234
exists(ArgumentRoutingConfig cfg | cfg.hasFlow(source, sink))
3335
select source, sink

0 commit comments

Comments
 (0)