Merge pull request #1103 from markshannon/python-encapsulate-builtins

Python: encapsulate extensionals dealing with 'builtin' objects.

Author: taus-semmle

python/ql/src/Classes/ClassAttributes.qll

@@ -130,7 +130,7 @@ class CheckClass extends ClassObject {
 
 
 private Object object_getattribute() {
-    py_cmembers_versioned(theObjectType(), "__getattribute__", result, major_version().toString())
+    result.asBuiltin() = theObjectType().asBuiltin().getMember("__getattribute__")
 }
 
 private predicate auto_name(string name) {

python/ql/src/semmle/python/Import.qll

@@ -1,5 +1,5 @@
 import python
-
+private import semmle.python.types.Builtins
 
 /** An alias in an import statement, the `mod as name` part of `import mod as name`. May be artificial;
     `import x` is transformed into `import x as x` */
@@ -14,7 +14,7 @@ class Alias extends Alias_ {
 private predicate valid_module_name(string name) {
     exists(Module m | m.getName() = name)
     or
-    exists(Object cmod | py_cobjecttypes(cmod, theModuleType()) and py_cobjectnames(cmod, name))
+    exists(Builtin cmod | cmod.getClass() = theModuleType().asBuiltin() and cmod.getName() = name)
 }
 
 /** An artificial expression representing an import */

python/ql/src/semmle/python/pointsto/Base.qll

@@ -9,6 +9,7 @@
  */
 import python
 import semmle.python.dataflow.SsaDefinitions
+private import semmle.python.types.Builtins
 
 module BasePointsTo {
     /** INTERNAL -- Use n.refersTo(value, _, origin) instead */
@@ -49,7 +50,7 @@ ClassObject simple_types(Object obj) {
     or
     obj.getOrigin() instanceof Module and result = theModuleType()
     or
-    result = builtin_object_type(obj)
+    result.asBuiltin() = obj.asBuiltin().getClass()
     or
     obj = unknownValue() and result = theUnknownType()
 }
@@ -126,34 +127,6 @@ predicate baseless_is_new_style(ClassObject cls) {
  * analysis.
  */
 
-/** Gets the base class of built-in class `cls` */
-pragma [noinline]
-ClassObject builtin_base_type(ClassObject cls) {
-    /* The extractor uses the special name ".super." to indicate the super class of a builtin class */
-    py_cmembers_versioned(cls, ".super.", result, _)
-}
-
-/** Gets the `name`d attribute of built-in class `cls` */
-pragma [noinline]
-Object builtin_class_attribute(ClassObject cls, string name) {
-    not name = ".super." and
-    py_cmembers_versioned(cls, name, result, _)
-}
-
-/** Holds if the `name`d attribute of built-in module `m` is `value` of `cls` */
-pragma [noinline]
-predicate builtin_module_attribute(ModuleObject m, string name, Object value, ClassObject cls) {
-    py_cmembers_versioned(m, name, value, _) and cls = builtin_object_type(value)
-}
-
-/** Gets the (built-in) class of the built-in object `obj` */
-pragma [noinline]
-ClassObject builtin_object_type(Object obj) {
-    py_cobjecttypes(obj, result) and not obj = unknownValue()
-    or
-    obj = unknownValue() and result = theUnknownType()
-}
-
 /** Holds if this class (not on a super-class) declares name */
 pragma [noinline]
 predicate class_declares_attribute(ClassObject cls, string name) {
@@ -162,11 +135,11 @@ predicate class_declares_attribute(ClassObject cls, string name) {
         class_defines_name(defn, name)
     )
     or
-    exists(Object o |
-        o = builtin_class_attribute(cls, name) and 
-        not exists(ClassObject sup |
-            sup = builtin_base_type(cls) and
-            o = builtin_class_attribute(sup, name)
+    exists(Builtin o |
+        o = cls.asBuiltin().getMember(name) and 
+        not exists(Builtin sup |
+            sup = cls.asBuiltin().getBaseClass() and
+            o = sup.getMember(name)
         )
     )
 }
@@ -558,11 +531,11 @@ Object undefinedVariable() {
 
 /** Gets the pseudo-object representing an unknown value */
 Object unknownValue() {
-    py_special_objects(result, "_1")
+    result.asBuiltin() = Builtin::unknown()
 }
 
 BuiltinCallable theTypeNewMethod() {
-    py_cmembers_versioned(theTypeType(), "__new__", result, major_version().toString())
+    result.asBuiltin() = theTypeType().asBuiltin().getMember("__new__")
 }
 
 /** Gets the `value, cls, origin` that `f` would refer to if it has not been assigned some other value */
@@ -578,7 +551,7 @@ predicate potential_builtin_points_to(NameNode f, Object value, ClassObject cls,
 
 pragma [noinline]
 predicate builtin_name_points_to(string name, Object value, ClassObject cls) {
-    value = Object::builtin(name) and py_cobjecttypes(value, cls)
+    value = Object::builtin(name) and cls.asBuiltin() = value.asBuiltin().getClass()
 }
 
 module BaseFlow {

python/ql/src/semmle/python/pointsto/PointsTo.qll

@@ -27,6 +27,7 @@ import python
 private import PointsToContext
 private import Base
 private import semmle.python.types.Extensions
+private import semmle.python.types.Builtins
 private import Filters as BaseFilters
 import semmle.dataflow.SSA
 private import MRO
@@ -311,7 +312,7 @@ module PointsTo {
               exists(SubscriptNode sub, Object sys_modules |
                   sub.getValue() = sys_modules_flow and
                   points_to(sys_modules_flow, _, sys_modules, _, _) and
-                  builtin_module_attribute(theSysModuleObject(), "modules", sys_modules, _) and
+                  sys_modules.asBuiltin() = Builtin::special("sys").getMember("modules") and
                   sub.getIndex() = n and
                   n.getNode().(StrConst).getText() = name and
                   sub.(DefinitionNode).getValue() = mod and
@@ -435,7 +436,7 @@ module PointsTo {
     }
 
     private boolean module_exports_boolean(ModuleObject mod, string name) {
-        py_cmembers_versioned(mod, name, _, major_version().toString()) and
+        exists(mod.asBuiltin().getMember(name)) and
         name.charAt(0) != "_" and result = true
         or
         result = package_exports_boolean(mod, name)
@@ -494,7 +495,8 @@ module PointsTo {
             or
             package_attribute_points_to(mod, name, value, cls, origin)
             or
-            builtin_module_attribute(mod, name, value, cls) and origin = CfgOrigin::unknown()
+            value.asBuiltin() = mod.asBuiltin().getMember(name) and
+            cls.asBuiltin() = value.asBuiltin().getClass() and origin = CfgOrigin::unknown()
         }
 
     }
@@ -2458,7 +2460,7 @@ module PointsTo {
                 is_new_style(cls) and not exists(cls_expr.getBase(0)) and result = theObjectType() and n = 0
             )
             or
-            result = builtin_base_type(cls) and n = 0
+            result.asBuiltin() = cls.asBuiltin().getBaseClass() and n = 0
             or
             cls = theUnknownType() and result = theObjectType() and n = 0
         }
@@ -2482,7 +2484,7 @@ module PointsTo {
             or
             cls = theObjectType() and result = 0
             or
-            exists(builtin_base_type(cls)) and cls != theObjectType() and result = 1
+            exists(cls.asBuiltin().getBaseClass()) and cls != theObjectType() and result = 1
             or
             cls = theUnknownType() and result = 1
         }
@@ -2646,8 +2648,8 @@ module PointsTo {
                 ssa_variable_points_to(var, _, value, vcls, origin)
             )
             or
-            value = builtin_class_attribute(owner, name) and class_declares_attribute(owner, name) and
-            origin = CfgOrigin::unknown() and vcls = builtin_object_type(value)
+            value.asBuiltin() = owner.asBuiltin().getMember(name) and class_declares_attribute(owner, name) and
+            origin = CfgOrigin::unknown() and vcls.asBuiltin() = value.asBuiltin().getClass()
         }
 
         private predicate interesting_class_attribute(ClassList mro, string name) {
@@ -2754,7 +2756,11 @@ module PointsTo {
                 obj = unknownValue() and result = theUnknownType()
             )
             or
-            py_cobjecttypes(cls, result) and is_c_metaclass(result)
+            exists(Builtin meta |
+                result.asBuiltin() = meta and
+                meta = cls.asBuiltin().getClass() and
+                meta.inheritsFromType()
+            )
             or
             exists(ControlFlowNode meta |
                 Types::six_add_metaclass(_, cls, meta) and
@@ -2777,7 +2783,7 @@ module PointsTo {
         }
 
         private boolean has_declared_metaclass(ClassObject cls) {
-            py_cobjecttypes(cls, _) and result = true
+            exists(cls.asBuiltin().getClass()) and result = true
             or
             result = has_six_add_metaclass(cls).booleanOr(has_metaclass_var_metaclass(cls))
         }

python/ql/src/semmle/python/types/Builtins.qll

@@ -0,0 +1,110 @@
+import python
+
+class Builtin extends @py_cobject {
+
+    Builtin() {
+        not (
+            /* @py_cobjects for modules which have a corresponding Python module */
+            exists(@py_cobject mod_type | py_special_objects(mod_type, "ModuleType") and py_cobjecttypes(this, mod_type)) and
+            exists(Module m | py_cobjectnames(this, m.getName()))
+        )
+        and (
+            /* Exclude unmatched builtin objects in the library trap files */
+            py_cobjectnames(this, _) or
+            py_cobjecttypes(this, _) or
+            py_special_objects(this, _)
+        )
+    }
+
+    string toString() {
+        not this = undefinedVariable().asBuiltin() and not this = Builtin::unknown() and
+        exists(Builtin type, string typename, string objname |
+            py_cobjecttypes(this, type) and py_cobjectnames(this, objname) and typename = type.getName() |
+            result = typename + " " + objname
+        )
+    }
+
+    Builtin getClass() {
+        py_cobjecttypes(this, result) and not this = Builtin::unknown()
+        or
+        this = Builtin::unknown() and result = Builtin::unknownType()
+    }
+
+    Builtin getMember(string name) {
+        not name = ".super." and
+        py_cmembers_versioned(this, name, result, major_version().toString())
+    }
+
+    Builtin getItem(int index) {
+        py_citems(this, index, result)
+    }
+
+    Builtin getBaseClass() {
+        /* The extractor uses the special name ".super." to indicate the super class of a builtin class */
+        py_cmembers_versioned(this, ".super.", result, major_version().toString())
+    }
+
+    predicate inheritsFromType() {
+        this = Builtin::special("type")
+        or
+        this.getBaseClass().inheritsFromType()
+    }
+
+    string getName() {
+        py_cobjectnames(this, result)
+    }
+
+    predicate isClass() {
+        py_cobjecttypes(_, this) or
+        this = Builtin::unknownType() or
+        exists(Builtin meta | meta.inheritsFromType() and py_cobjecttypes(this, meta))
+    }
+
+    predicate isFunction() {
+        this.getClass() = Builtin::special("BuiltinFunctionType") and
+        exists(Builtin mod |
+            mod.isModule() and
+            mod.getMember(_) = this
+        )
+    }
+
+    predicate isModule() {
+        this.getClass() = Builtin::special("ModuleType")
+    }
+
+    predicate isMethod() {
+        this.getClass() = Builtin::special("MethodDescriptorType")
+        or
+        this.getClass() = Builtin::special("BuiltinFunctionType") and
+        exists(Builtin cls | cls.isClass() and cls.getMember(_) = this)
+        or
+        this.getClass().getName() = "wrapper_descriptor"
+    }
+
+}
+
+module Builtin {
+
+    Builtin builtinModule() {
+        py_special_objects(result, "builtin_module_2") and major_version() = 2
+        or
+        py_special_objects(result, "builtin_module_3") and major_version() = 3
+    }
+
+    Builtin builtin(string name) {
+        result = builtinModule().getMember(name)
+    }
+
+    Builtin special(string name) {
+        py_special_objects(result, name)
+    }
+
+    Builtin unknown() {
+        py_special_objects(result, "_1")
+    }
+
+    Builtin unknownType() {
+        py_special_objects(result, "_semmle_unknown_type")
+    }
+
+}