Merge pull request #1389 from markshannon/python-ipa-objects-fix-performance

Python: New points-to and object model with performance fixes

Author: taus-semmle

python/ql/src/Exceptions/UnguardedNextInGenerator.ql

@@ -16,7 +16,7 @@ FunctionObject iter() {
     result = Object::builtin("iter")
 }
 
-FunctionObject next() {
+BuiltinFunctionObject next() {
     result = Object::builtin("next")
 }
 

python/ql/src/Expressions/IsComparisons.qll

@@ -104,8 +104,14 @@ predicate invalid_portable_is_comparison(Compare comp, Cmpop op, ClassObject cls
     /* OK to use 'is' when comparing items from a known set of objects */
     not exists(Expr left, Expr right, Object obj |
         comp.compares(left, op, right) and
-        left.refersTo(obj) and right.refersTo(obj) and
-        exists(ImmutableLiteral il | il.getLiteralObject() = obj)
+        exists(ImmutableLiteral il | il.getLiteralObject() = obj) |
+        left.refersTo(obj) and right.refersTo(obj)
+        or
+        /* Simple constant in module, probably some sort of sentinel */
+        exists(AstNode origin |
+            not left.refersTo(_) and right.refersTo(obj, origin) and
+            origin.getScope().getEnclosingModule() = comp.getScope().getEnclosingModule()
+        )
     )
     and
     /* OK to use 'is' when comparing with a member of an enum */

python/ql/src/Security/CWE-798/HardcodedCredentials.ql

@@ -53,6 +53,10 @@ predicate capitalized_word(StrConst str) {
     str.getText().regexpMatch("[A-Z][a-z]+")
 }
 
+predicate format_string(StrConst str) {
+    str.getText().matches("%{%}%")
+}
+
 predicate maybeCredential(ControlFlowNode f) {
     /* A string that is not too short and unlikely to be text or an identifier. */
     exists(StrConst str |
@@ -66,20 +70,21 @@ predicate maybeCredential(ControlFlowNode f) {
         /* Not too repetitive */
         exists(int chars |
             chars = char_count(str) |
-            chars > 20 or
-            chars > str.getText().length()/2
+            chars > 15 or
+            chars*3 > str.getText().length()*2
         ) and
         not possible_reflective_name(str.getText()) and
-        not capitalized_word(str)
+        not capitalized_word(str) and
+        not format_string(str)
     )
     or
-    /* Or, an integer with at least 8 digits */
+    /* Or, an integer with over 32 bits */
     exists(IntegerLiteral lit |
         f.getNode() = lit
         |
-        not exists(lit.getValue())
-        or
-        lit.getValue() > 10000000
+        not exists(lit.getValue()) and
+        /* Not a set of flags or round number */
+        not lit.getN().matches("%00%")
     )
 }
 

python/ql/src/analysis/CallGraphEfficiency.ql

@@ -9,15 +9,15 @@ import semmle.python.pointsto.PointsToContext
 
 from int total_facts, int total_size, int depth, float efficiency
 where 
-total_facts = strictcount(ControlFlowNode call, FunctionObject func |
+total_facts = strictcount(ControlFlowNode call, CallableValue func |
     exists(PointsToContext ctx |
-        call = PointsTo::get_a_call(func, ctx) and
+        call = func.getACall(ctx) and
         depth = ctx.getDepth()
     )
 )
 and
-total_size = strictcount(ControlFlowNode call, FunctionObject func, PointsToContext ctx |
-    call = PointsTo::get_a_call(func, ctx) and
+total_size = strictcount(ControlFlowNode call, CallableValue func, PointsToContext ctx |
+    call = func.getACall(ctx) and
     depth = ctx.getDepth()
 )
 and

python/ql/src/analysis/CallGraphMarginalEfficiency.ql

@@ -8,20 +8,20 @@ import semmle.python.pointsto.PointsToContext
 
 from int total_facts, int total_size, int depth, float efficiency
 where 
-total_facts = strictcount(ControlFlowNode call, FunctionObject func |
+total_facts = strictcount(ControlFlowNode call, CallableValue func |
     exists(PointsToContext ctx |
-        call = PointsTo::get_a_call(func, ctx) and
+        call = func.getACall(ctx) and
         depth = ctx.getDepth()
         and not
         exists(PointsToContext shallower |
-            call = PointsTo::get_a_call(func, shallower) and
+        call = func.getACall(shallower) and
             shallower.getDepth() < depth
         )
     )
 )
 and
-total_size = strictcount(ControlFlowNode call, FunctionObject func, PointsToContext ctx |
-    call = PointsTo::get_a_call(func, ctx) and
+total_size = strictcount(ControlFlowNode call, CallableValue func, PointsToContext ctx |
+    call = func.getACall(ctx) and
     depth = ctx.getDepth()
 )
 and

python/ql/src/analysis/ContextMarginalEfficiency.ql

@@ -29,4 +29,4 @@ total_size = strictcount(ControlFlowNode f, Object value, ClassObject cls, Point
 )
 and
 efficiency = 100.0 * total_facts / total_size
-select depth, total_facts, total_size, efficiency
+select depth, total_facts, total_size, efficiency

python/ql/src/analysis/FailedInference.ql

@@ -2,10 +2,10 @@
 import python
 import semmle.python.pointsto.PointsTo
 
-from ClassObject cls, string reason
+from ClassValue cls, string reason
 
 where
-PointsTo::Types::failed_inference(cls, reason)
+Types::failedInference(cls, reason)
 
 select cls, reason
 

python/ql/src/analysis/KeyPointsToFailure.ql

@@ -8,11 +8,12 @@
  */
 
 import python
+import semmle.python.pointsto.PointsTo
 
 predicate points_to_failure(Expr e) {
-    exists(ControlFlowNode f | 
+    exists(ControlFlowNode f |
         f = e.getAFlowNode() |
-        not f.refersTo(_)
+        not PointsTo::pointsTo(f, _, _, _)
     )
 }
 

python/ql/src/analysis/Pruned.ql

@@ -6,7 +6,7 @@ from int size
 
 where
 size = count(ControlFlowNode f |
-    not PointsTo::Test::reachableBlock(f.getBasicBlock(), _)
+    not PointsToInternal::reachableBlock(f.getBasicBlock(), _)
 )
 
 

python/ql/src/python.qll

@@ -36,5 +36,6 @@ import semmle.dataflow.SSA
 import semmle.python.pointsto.Base
 import semmle.python.pointsto.Context
 import semmle.python.pointsto.CallGraph
+import semmle.python.objects.ObjectAPI
 
 import site