Merge pull request #9575 from erik-krogh/paramModules

QL: support for parameterized modules

Author: erik-krogh

ql/Cargo.lock

@@ -10,7 +10,7 @@ edition = "2018"
 flate2 = "1.0"
 node-types = { path = "../node-types" }
 tree-sitter = "0.19"
-tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "c3d626a77cf5acc5d8c11aaf91c12348880c8eca" }
+tree-sitter-ql = { git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "343cc5873e20510586ade803659ef8ce153bd603" }
 clap = "2.33"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }

ql/extractor/Cargo.toml

@@ -11,4 +11,4 @@ clap = "2.33"
 node-types = { path = "../node-types" }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
-tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "c3d626a77cf5acc5d8c11aaf91c12348880c8eca" }
+tree-sitter-ql = { git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "343cc5873e20510586ade803659ef8ce153bd603" }

ql/generator/Cargo.toml

@@ -162,6 +162,8 @@ class QLDoc extends TQLDoc, AstNode {
   string getContents() { result = qldoc.getValue() }
 
   override string getAPrimaryQlClass() { result = "QLDoc" }
+
+  override AstNode getParent() { result.getQLDoc() = this }
 }
 
 class BlockComment extends TBlockComment, AstNode {
@@ -638,7 +640,7 @@ class FieldDecl extends TFieldDecl, AstNode {
 /**
  * A type reference, such as `DataFlow::Node`.
  */
-class TypeExpr extends TType, AstNode {
+class TypeExpr extends TType, TypeRef {
   QL::TypeExpr type;
 
   TypeExpr() { this = TType(type) }
@@ -675,10 +677,14 @@ class TypeExpr extends TType, AstNode {
    */
   ModuleExpr getModule() { toQL(result) = type.getQualifier() }
 
-  /**
-   * Gets the type that this type reference refers to.
-   */
-  Type getResolvedType() { resolveTypeExpr(this, result) }
+  /** Gets the type that this type reference refers to. */
+  override Type getResolvedType() {
+    // resolve type
+    resolveTypeExpr(this, result)
+    or
+    // if it resolves to a module
+    exists(FileOrModule mod | resolveModuleRef(this, mod) | result = mod.toType())
+  }
 
   override AstNode getAChild(string pred) {
     result = super.getAChild(pred)
@@ -710,6 +716,9 @@ class Module extends TModule, ModuleDeclaration {
     exists(int i | result = this.getMember(i) and m = this.getMember(i + 1))
   }
 
+  /** Gets a ref to the module that this module implements. */
+  TypeExpr getImplements(int i) { toQL(result) = mod.getImplements(i).getTypeExpr() }
+
   /** Gets the module expression that this module is an alias for, if any. */
   ModuleExpr getAlias() { toQL(result) = mod.getAFieldOrChild().(QL::ModuleAliasBody).getChild() }
 
@@ -719,6 +728,19 @@ class Module extends TModule, ModuleDeclaration {
     pred = directMember("getAlias") and result = this.getAlias()
     or
     pred = directMember("getAMember") and result = this.getAMember()
+    or
+    exists(int i | pred = indexedMember("getImplements", i) and result = this.getImplements(i))
+    or
+    exists(int i | pred = indexedMember("hasParameter", i) and this.hasParameter(i, _, result))
+  }
+
+  /** Holds if the `i`th parameter of this module has `name` and type `sig`. */
+  predicate hasParameter(int i, string name, SignatureExpr sig) {
+    exists(QL::ModuleParam param |
+      param = mod.getParameter(i) and
+      name = param.getParameter().getValue() and
+      sig.toQL() = param.getSignature()
+    )
   }
 }
 
@@ -1093,16 +1115,18 @@ class InlineCast extends TInlineCast, Expr {
   }
 }
 
-/** An entity that resolves to a module. */
-class ModuleRef extends AstNode, TModuleRef {
-  /** Gets the module that this entity resolves to. */
-  FileOrModule getResolvedModule() { none() }
+/** An entity that resolves to a type. */
+class TypeRef extends AstNode, TTypeRef {
+  abstract Type getResolvedType();
+
+  /** Gets the module that this entity resolves to, if this reference resolves to a module */
+  final FileOrModule getResolvedModule() { result.toType() = this.getResolvedType() }
 }
 
 /**
  * An import statement.
  */
-class Import extends TImport, ModuleMember, ModuleRef {
+class Import extends TImport, ModuleMember, TypeRef {
   QL::ImportDirective imp;
 
   Import() { this = TImport(imp) }
@@ -1153,7 +1177,9 @@ class Import extends TImport, ModuleMember, ModuleRef {
     )
   }
 
-  final override FileOrModule getResolvedModule() { resolve(this, result) }
+  override Type getResolvedType() {
+    exists(FileOrModule mod | resolve(this, mod) | result = mod.toType())
+  }
 }
 
 /** A formula, such as `x = 6 and y < 5`. */
@@ -2172,7 +2198,7 @@ class DontCare extends TDontCare, Expr {
 }
 
 /** A module expression. Such as `DataFlow` in `DataFlow::Node` */
-class ModuleExpr extends TModuleExpr, ModuleRef {
+class ModuleExpr extends TModuleExpr, TypeRef {
   QL::ModuleExpr me;
 
   ModuleExpr() { this = TModuleExpr(me) }
@@ -2190,6 +2216,10 @@ class ModuleExpr extends TModuleExpr, ModuleRef {
     result = me.getName().(QL::SimpleId).getValue()
     or
     not exists(me.getName()) and result = me.getChild().(QL::SimpleId).getValue()
+    or
+    exists(QL::ModuleInstantiation instantiation | instantiation.getParent() = me |
+      result = instantiation.getName().getChild().getValue()
+    )
   }
 
   /**
@@ -2203,7 +2233,9 @@ class ModuleExpr extends TModuleExpr, ModuleRef {
    */
   ModuleExpr getQualifier() { result = TModuleExpr(me.getChild()) }
 
-  final override FileOrModule getResolvedModule() { resolveModuleExpr(this, result) }
+  override Type getResolvedType() {
+    exists(FileOrModule mod | resolveModuleRef(this, mod) | result = mod.toType())
+  }
 
   final override string toString() { result = this.getName() }
 
@@ -2213,9 +2245,41 @@ class ModuleExpr extends TModuleExpr, ModuleRef {
     result = super.getAChild(pred)
     or
     pred = directMember("getQualifier") and result = this.getQualifier()
+    or
+    exists(int i | pred = indexedMember("getArgument", i) and result = this.getArgument(i))
+  }
+
+  /**
+   * Gets the `i`th type argument if this module is a module instantiation.
+   * The result is either a `PredicateExpr` or a `TypeExpr`.
+   */
+  SignatureExpr getArgument(int i) {
+    exists(QL::ModuleInstantiation instantiation | instantiation.getParent() = me |
+      result.toQL() = instantiation.getChild(i)
+    )
   }
 }
 
+/** A signature expression, either a `PredicateExpr` or a `TypeExpr`. */
+class SignatureExpr extends TSignatureExpr, AstNode {
+  QL::SignatureExpr sig;
+
+  SignatureExpr() {
+    toQL(this) = sig.getPredicate()
+    or
+    toQL(this) = sig.getTypeExpr()
+  }
+
+  /** Gets the generated AST node that contains this signature expression. */
+  QL::SignatureExpr toQL() { result = sig }
+
+  /** Gets this signature expression if it represents a predicate expression. */
+  PredicateExpr asPredicate() { result = this }
+
+  /** Gets this signature expression if it represents a type expression. */
+  TypeExpr asType() { result = this }
+}
+
 /** An argument to an annotation. */
 private class AnnotationArg extends TAnnotationArg, AstNode {
   QL::AnnotArg arg;
@@ -2272,7 +2336,7 @@ class Annotation extends TAnnotation, AstNode {
   /** Gets the node corresponding to the field `name`. */
   string getName() { result = annot.getName().getValue() }
 
-  override AstNode getParent() { result = AstNode.super.getParent() }
+  override AstNode getParent() { result.getAnAnnotation() = this }
 
   override AstNode getAChild(string pred) {
     result = super.getAChild(pred)

ql/ql/src/codeql_ql/ast/Ast.qll

@@ -83,10 +83,12 @@ class TExpr =
 
 class TCall = TPredicateCall or TMemberCall or TNoneCall or TAnyCall;
 
-class TModuleRef = TImport or TModuleExpr;
+class TTypeRef = TImport or TModuleExpr or TType;
 
 class TYamlNode = TYamlCommemt or TYamlEntry or TYamlKey or TYamlListitem or TYamlValue;
 
+class TSignatureExpr = TPredicateExpr or TType;
+
 /** DEPRECATED: Alias for TYamlNode */
 deprecated class TYAMLNode = TYamlNode;
 
@@ -229,4 +231,6 @@ module AstConsistency {
     not node instanceof YAML::YAMLNode and // parents for YAML doens't work
     not (node instanceof QLDoc and node.getLocation().getFile().getExtension() = "dbscheme") // qldoc in dbschemes are not hooked up
   }
+
+  query predicate nonUniqueParent(AstNode node) { count(node.getParent()) >= 2 }
 }

ql/ql/src/codeql_ql/ast/internal/AstNodes.qll

@@ -23,24 +23,30 @@ private class ContainerOrModule extends TContainerOrModule {
     or
     this = TFolder(_) and result = "folder"
   }
-}
-
-private class TFileOrModule = TFile or TModule;
 
-/** A file or a module. */
-class FileOrModule extends TFileOrModule, ContainerOrModule {
   /** Gets the module for this imported module. */
   Module asModule() { this = TModule(result) }
 
   /** Gets the file for this file. */
   File asFile() { this = TFile(result) }
+}
+
+private class TFileOrModule = TFile or TModule;
 
+/** A file or a module. */
+class FileOrModule extends TFileOrModule, ContainerOrModule {
   /** Gets the file that contains this module/file. */
   File getFile() {
     result = this.asFile()
     or
     result = this.asModule().getLocation().getFile()
   }
+
+  Type toType() {
+    result.(FileType).getDeclaration().getLocation().getFile() = this.asFile()
+    or
+    result.(ModuleType).getDeclaration() = this.asModule()
+  }
 }
 
 private class File_ extends FileOrModule, TFile {
@@ -207,31 +213,34 @@ private module Cached {
 
   /** Holds if module expression `me` resolves to `m`. */
   cached
-  predicate resolveModuleExpr(ModuleExpr me, FileOrModule m) {
+  predicate resolveModuleRef(TypeRef me, FileOrModule m) {
     not m = TFile(any(File f | f.getExtension() = "ql")) and
-    not exists(me.getQualifier()) and
-    exists(ContainerOrModule enclosing, string name | resolveModuleExprHelper(me, enclosing, name) |
+    not exists(me.(ModuleExpr).getQualifier()) and
+    exists(ContainerOrModule enclosing, string name | resolveModuleRefHelper(me, enclosing, name) |
       definesModule(enclosing, name, m, _)
     )
     or
     exists(FileOrModule mid |
-      resolveModuleExpr(me.getQualifier(), mid) and
-      definesModule(mid, me.getName(), m, true)
+      resolveModuleRef(me.(ModuleExpr).getQualifier(), mid) and
+      definesModule(mid, me.(ModuleExpr).getName(), m, true)
     )
   }
 
   pragma[noinline]
-  private predicate resolveModuleExprHelper(ModuleExpr me, ContainerOrModule enclosing, string name) {
+  private predicate resolveModuleRefHelper(TypeRef me, ContainerOrModule enclosing, string name) {
     enclosing = getEnclosingModule(me).getEnclosing*() and
-    name = me.getName()
+    name = [me.(ModuleExpr).getName(), me.(TypeExpr).getClassName()]
   }
 }
 
 import Cached
 private import NewType
 
 boolean getPublicBool(AstNode n) {
-  if n.(ModuleMember).isPrivate() or n.(NewTypeBranch).getNewType().isPrivate()
+  if
+    n.(ModuleMember).isPrivate() or
+    n.(NewTypeBranch).getNewType().isPrivate() or
+    n.(Module).isPrivate()
   then result = false
   else result = true
 }
@@ -253,6 +262,22 @@ private predicate definesModule(
     m = TModule(any(Module mod | public = getPublicBool(mod)))
   )
   or
+  // signature module in a paramertized module
+  exists(Module mod, SignatureExpr sig, TypeExpr ty, int i |
+    mod = container.asModule() and
+    mod.hasParameter(i, name, sig) and
+    public = false and
+    ty = sig.asType()
+  |
+    // resolve to the signature module
+    m = ty.getResolvedModule()
+    or
+    // resolve to the arguments of the instantiated module
+    exists(ModuleExpr inst | inst.getResolvedModule().asModule() = mod |
+      m = inst.getArgument(i).asType().getResolvedModule()
+    )
+  )
+  or
   // import X
   exists(Import imp, ContainerOrModule m0 |
     container = getEnclosingModule(imp) and
@@ -274,7 +299,7 @@ private predicate definesModule(
   exists(Module alias |
     container = getEnclosingModule(alias) and
     name = alias.getName() and
-    resolveModuleExpr(alias.getAlias(), m) and
+    resolveModuleRef(alias.getAlias(), m) and
     public = getPublicBool(alias)
   )
 }
@@ -288,14 +313,6 @@ module ModConsistency {
         .regexpMatch(".*/(test|examples|ql-training|recorded-call-graph-metrics)/.*")
   }
 
-  query predicate noResolveModuleExpr(ModuleExpr me) {
-    not resolveModuleExpr(me, _) and
-    not me.getLocation()
-        .getFile()
-        .getAbsolutePath()
-        .regexpMatch(".*/(test|examples|ql-training|recorded-call-graph-metrics)/.*")
-  }
-
   query predicate multipleResolve(Import imp, int c, ContainerOrModule m) {
     c = strictcount(ContainerOrModule m0 | resolve(imp, m0)) and
     c > 1 and
@@ -306,9 +323,16 @@ module ModConsistency {
         .regexpMatch(".*/(test|examples|ql-training|recorded-call-graph-metrics)/.*")
   }
 
-  query predicate multipleResolveModuleExpr(ModuleExpr me, int c, ContainerOrModule m) {
-    c = strictcount(ContainerOrModule m0 | resolveModuleExpr(me, m0)) and
-    c > 1 and
-    resolveModuleExpr(me, m)
-  }
+  // This can happen with parameterized modules.
+  /*
+   * query predicate multipleResolveModuleRef(ModuleExpr me, int c, ContainerOrModule m) {
+   *    c = strictcount(ContainerOrModule m0 | resolveModuleRef(me, m0)) and
+   *    c > 1 and
+   *    resolveModuleRef(me, m)
+   *  }
+   */
+
+  query predicate noName(Module mod) { not exists(mod.getName()) }
+
+  query predicate nonUniqueName(Module mod) { count(mod.getName()) >= 2 }
 }