Merge pull request #5641 from tausbn/python-use-localsourcenode-in-typetrackers

Python: Use API graphs in PEP249 support

Author: yoff

python/change-notes/2021-04-13-pep249-api-graphs.md

@@ -0,0 +1,4 @@
+lgtm,codescanning
+* Modeling of libraries supporting PEP249 has been changed to use API graphs. When defining new
+  models, the relevant extension point is now `PEP249ModuleApiNode` in the `PEP249` module, instead
+  of `PEP249Module`. The latter class has now been deprecated.

python/ql/src/semmle/python/frameworks/Django.qll

@@ -355,8 +355,11 @@ private module PrivateDjango {
     /** Gets a reference to the `django.db` module. */
     DataFlow::Node db() { result = django_attr("db") }
 
-    class DjangoDb extends PEP249Module {
-      DjangoDb() { this = db() }
+    /**
+     * `django.db` implements PEP249, providing ways to execute SQL statements against a database.
+     */
+    private class DjangoDb extends PEP249ModuleApiNode {
+      DjangoDb() { this = API::moduleImport("django").getMember("db") }
     }
 
     /** Provides models for the `django.db` module. */

python/ql/src/semmle/python/frameworks/MySQLdb.qll

@@ -9,6 +9,7 @@ private import python
 private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
 private import PEP249
 
 /**
@@ -21,19 +22,8 @@ private module MySQLdb {
   // ---------------------------------------------------------------------------
   // MySQLdb
   // ---------------------------------------------------------------------------
-  /** Gets a reference to the `MySQLdb` module. */
-  private DataFlow::Node moduleMySQLdb(DataFlow::TypeTracker t) {
-    t.start() and
-    result = DataFlow::importNode("MySQLdb")
-    or
-    exists(DataFlow::TypeTracker t2 | result = moduleMySQLdb(t2).track(t2, t))
-  }
-
-  /** Gets a reference to the `MySQLdb` module. */
-  DataFlow::Node moduleMySQLdb() { result = moduleMySQLdb(DataFlow::TypeTracker::end()) }
-
   /** MySQLdb implements PEP 249, providing ways to execute SQL statements against a database. */
-  class MySQLdb extends PEP249Module {
-    MySQLdb() { this = moduleMySQLdb() }
+  class MySQLdb extends PEP249ModuleApiNode {
+    MySQLdb() { this = API::moduleImport("MySQLdb") }
   }
 }

python/ql/src/semmle/python/frameworks/MysqlConnectorPython.qll

@@ -9,6 +9,7 @@ private import python
 private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
 private import PEP249
 
 /**
@@ -21,64 +22,14 @@ private module MysqlConnectorPython {
   // ---------------------------------------------------------------------------
   // mysql
   // ---------------------------------------------------------------------------
-  /** Gets a reference to the `mysql` module. */
-  private DataFlow::Node mysql(DataFlow::TypeTracker t) {
-    t.start() and
-    result = DataFlow::importNode("mysql")
-    or
-    exists(DataFlow::TypeTracker t2 | result = mysql(t2).track(t2, t))
-  }
-
-  /** Gets a reference to the `mysql` module. */
-  DataFlow::Node mysql() { result = mysql(DataFlow::TypeTracker::end()) }
-
-  /**
-   * Gets a reference to the attribute `attr_name` of the `mysql` module.
-   * WARNING: Only holds for a few predefined attributes.
-   */
-  private DataFlow::Node mysql_attr(DataFlow::TypeTracker t, string attr_name) {
-    attr_name in ["connector"] and
-    (
-      t.start() and
-      result = DataFlow::importNode("mysql" + "." + attr_name)
-      or
-      t.startInAttr(attr_name) and
-      result = mysql()
-    )
-    or
-    // Due to bad performance when using normal setup with `mysql_attr(t2, attr_name).track(t2, t)`
-    // we have inlined that code and forced a join
-    exists(DataFlow::TypeTracker t2 |
-      exists(DataFlow::StepSummary summary |
-        mysql_attr_first_join(t2, attr_name, result, summary) and
-        t = t2.append(summary)
-      )
-    )
-  }
-
-  pragma[nomagic]
-  private predicate mysql_attr_first_join(
-    DataFlow::TypeTracker t2, string attr_name, DataFlow::Node res, DataFlow::StepSummary summary
-  ) {
-    DataFlow::StepSummary::step(mysql_attr(t2, attr_name), res, summary)
-  }
-
-  /**
-   * Gets a reference to the attribute `attr_name` of the `mysql` module.
-   * WARNING: Only holds for a few predefined attributes.
-   */
-  private DataFlow::Node mysql_attr(string attr_name) {
-    result = mysql_attr(DataFlow::TypeTracker::end(), attr_name)
-  }
-
   /** Provides models for the `mysql` module. */
   module mysql {
     /**
      * The mysql.connector module
      * See https://dev.mysql.com/doc/connector-python/en/connector-python-example-connecting.html
      */
-    class MysqlConnector extends PEP249Module {
-      MysqlConnector() { this = mysql_attr("connector") }
+    class MysqlConnector extends PEP249ModuleApiNode {
+      MysqlConnector() { this = API::moduleImport("mysql").getMember("connector") }
     }
   }
 }

python/ql/src/semmle/python/frameworks/PEP249.qll

@@ -7,20 +7,26 @@ private import python
 private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
 
-/** A module implementing PEP 249. Extend this class for implementations. */
-abstract class PEP249Module extends DataFlow::Node { }
+/**
+ * A module implementing PEP 249. Extend this class for implementations.
+ *
+ * DEPRECATED: Extend `PEP249ModuleApiNode` instead.
+ */
+abstract deprecated class PEP249Module extends DataFlow::Node { }
 
-/** Gets a reference to a connect call. */
-private DataFlow::Node connect(DataFlow::TypeTracker t) {
-  t.startInAttr("connect") and
-  result instanceof PEP249Module
-  or
-  exists(DataFlow::TypeTracker t2 | result = connect(t2).track(t2, t))
+/**
+ * An abstract class encompassing API graph nodes that implement PEP 249.
+ * Extend this class for implementations.
+ */
+abstract class PEP249ModuleApiNode extends API::Node {
+  /** Gets a string representation of this element. */
+  override string toString() { result = this.(API::Node).toString() }
 }
 
 /** Gets a reference to a connect call. */
-DataFlow::Node connect() { result = connect(DataFlow::TypeTracker::end()) }
+DataFlow::Node connect() { result = any(PEP249ModuleApiNode a).getMember("connect").getAUse() }
 
 /**
  * Provides models for the `db.Connection` class
@@ -43,10 +49,8 @@ module Connection {
   abstract class InstanceSource extends DataFlow::Node { }
 
   /** A direct instantiation of `db.Connection`. */
-  private class ClassInstantiation extends InstanceSource, DataFlow::CfgNode {
-    override CallNode node;
-
-    ClassInstantiation() { node.getFunction() = connect().asCfgNode() }
+  private class ClassInstantiation extends InstanceSource, DataFlow::CallCfgNode {
+    ClassInstantiation() { this.getFunction() = connect() }
   }
 
   /** Gets a reference to an instance of `db.Connection`. */
@@ -115,10 +119,8 @@ private DataFlow::Node execute(DataFlow::TypeTracker t) {
 DataFlow::Node execute() { result = execute(DataFlow::TypeTracker::end()) }
 
 /** A call to the `execute` method on a cursor (or on a connection). */
-private class ExecuteCall extends SqlExecution::Range, DataFlow::CfgNode {
-  override CallNode node;
-
-  ExecuteCall() { node.getFunction() = execute().asCfgNode() }
+private class ExecuteCall extends SqlExecution::Range, DataFlow::CallCfgNode {
+  ExecuteCall() { this.getFunction() = execute() }
 
   override DataFlow::Node getSql() {
     result.asCfgNode() in [node.getArg(0), node.getArgByName("sql")]

python/ql/src/semmle/python/frameworks/Psycopg2.qll

@@ -9,6 +9,7 @@ private import python
 private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
 private import PEP249
 
 /**
@@ -21,19 +22,8 @@ private module Psycopg2 {
   // ---------------------------------------------------------------------------
   // Psycopg
   // ---------------------------------------------------------------------------
-  /** Gets a reference to the `psycopg2` module. */
-  private DataFlow::Node psycopg2(DataFlow::TypeTracker t) {
-    t.start() and
-    result = DataFlow::importNode("psycopg2")
-    or
-    exists(DataFlow::TypeTracker t2 | result = psycopg2(t2).track(t2, t))
-  }
-
-  /** Gets a reference to the `psycopg2` module. */
-  DataFlow::Node psycopg2() { result = psycopg2(DataFlow::TypeTracker::end()) }
-
   /** psycopg2 implements PEP 249, providing ways to execute SQL statements against a database. */
-  class Psycopg2 extends PEP249Module {
-    Psycopg2() { this = psycopg2() }
+  class Psycopg2 extends PEP249ModuleApiNode {
+    Psycopg2() { this = API::moduleImport("psycopg2") }
   }
 }

python/ql/src/semmle/python/frameworks/PyMySQL.qll

@@ -7,26 +7,16 @@ private import python
 private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
 private import PEP249
 
 /**
  * Provides models for the `PyMySQL` PyPI package.
  * See https://pypi.org/project/PyMySQL/
  */
 private module PyMySQL {
-  /** Gets a reference to the `pymysql` module. */
-  private DataFlow::Node pymysql(DataFlow::TypeTracker t) {
-    t.start() and
-    result = DataFlow::importNode("pymysql")
-    or
-    exists(DataFlow::TypeTracker t2 | result = pymysql(t2).track(t2, t))
-  }
-
-  /** Gets a reference to the `pymysql` module. */
-  DataFlow::Node pymysql() { result = pymysql(DataFlow::TypeTracker::end()) }
-
   /** PyMySQL implements PEP 249, providing ways to execute SQL statements against a database. */
-  class PyMySQLPEP249 extends PEP249Module {
-    PyMySQLPEP249() { this = pymysql() }
+  class PyMySQLPEP249 extends PEP249ModuleApiNode {
+    PyMySQLPEP249() { this = API::moduleImport("pymysql") }
   }
 }

python/ql/src/semmle/python/frameworks/Stdlib.qll

@@ -856,24 +856,13 @@ private module Stdlib {
   // ---------------------------------------------------------------------------
   // sqlite3
   // ---------------------------------------------------------------------------
-  /** Gets a reference to the `sqlite3` module. */
-  private DataFlow::Node sqlite3(DataFlow::TypeTracker t) {
-    t.start() and
-    result = DataFlow::importNode("sqlite3")
-    or
-    exists(DataFlow::TypeTracker t2 | result = sqlite3(t2).track(t2, t))
-  }
-
-  /** Gets a reference to the `sqlite3` module. */
-  DataFlow::Node sqlite3() { result = sqlite3(DataFlow::TypeTracker::end()) }
-
   /**
    * sqlite3 implements PEP 249, providing ways to execute SQL statements against a database.
    *
    * See https://devdocs.io/python~3.9/library/sqlite3
    */
-  class Sqlite3 extends PEP249Module {
-    Sqlite3() { this = sqlite3() }
+  class Sqlite3 extends PEP249ModuleApiNode {
+    Sqlite3() { this = API::moduleImport("sqlite3") }
   }
 }