Java: automodel application mode: use endpoint class like in framework mode

Author: Stephan Brandauer

java/ql/src/Telemetry/AutomodelApplicationModeCharacteristics.qll

@@ -21,21 +21,81 @@ import AutomodelEndpointTypes as AutomodelEndpointTypes
 
 newtype JavaRelatedLocationType = CallContext()
 
+newtype TApplicationModeEndpoint =
+  TExplicitArgument(Call call, DataFlow::Node arg) {
+    exists(Argument argExpr |
+      arg.asExpr() = argExpr and not argExpr.isVararg() and call = argExpr.getCall()
+    )
+  } or
+  TInstanceArgument(Call call, DataFlow::Node arg) { arg = DataFlow::getInstanceArgument(call) } or
+  TImplicitVarargsArray(Call call, DataFlow::ImplicitVarargsArray varargs, int idx) {
+    varargs.getCall() = call and
+    idx = min(Argument arg, int n | arg = call.getArgument(n) and arg.isVararg() | n)
+  }
+
+abstract private class ApplicationModeEndpoint extends TApplicationModeEndpoint {
+  abstract predicate isArgOf(Call c, int idx);
+
+  Call getCall() { this.isArgOf(result, _) }
+
+  int getArgIndex() { this.isArgOf(_, result) }
+
+  abstract Top asTop();
+
+  abstract DataFlow::Node asNode();
+
+  abstract string toString();
+}
+
 /**
  * A class representing nodes that are arguments to calls.
  */
-private class ArgumentNode extends DataFlow::Node {
-  Call c;
+class ExplicitArgument extends ApplicationModeEndpoint, TExplicitArgument {
+  Call call;
+  DataFlow::Node arg;
 
-  ArgumentNode() {
-    exists(Argument arg | this.asExpr() = arg and not arg.isVararg() and c = arg.getCall())
-    or
-    this.(DataFlow::ImplicitVarargsArray).getCall() = c
-    or
-    this = DataFlow::getInstanceArgument(c)
+  ExplicitArgument() { this = TExplicitArgument(call, arg) }
+
+  override predicate isArgOf(Call c, int idx) { c = call and this.asTop() = c.getArgument(idx) }
+
+  override Top asTop() { result = arg.asExpr() }
+
+  override DataFlow::Node asNode() { result = arg }
+
+  override string toString() { result = arg.toString() }
+}
+
+class InstanceArgument extends ApplicationModeEndpoint, TInstanceArgument {
+  Call call;
+  DataFlow::Node arg;
+
+  InstanceArgument() { this = TInstanceArgument(call, arg) }
+
+  override predicate isArgOf(Call c, int idx) {
+    c = call and this.asTop() = c.getQualifier() and idx = -1
   }
 
-  Call getCall() { result = c }
+  override Top asTop() { if exists(arg.asExpr()) then result = arg.asExpr() else result = call }
+
+  override DataFlow::Node asNode() { result = arg }
+
+  override string toString() { result = arg.toString() }
+}
+
+class ImplicitVarargsArray extends ApplicationModeEndpoint, TImplicitVarargsArray {
+  Call call;
+  DataFlow::ImplicitVarargsArray varargs;
+  int idx;
+
+  ImplicitVarargsArray() { this = TImplicitVarargsArray(call, varargs, idx) }
+
+  override predicate isArgOf(Call c, int i) { c = call and i = idx }
+
+  override Top asTop() { result = this.getCall() }
+
+  override DataFlow::Node asNode() { result = varargs }
+
+  override string toString() { result = varargs.toString() }
 }
 
 /**
@@ -47,7 +107,7 @@ private class ArgumentNode extends DataFlow::Node {
  */
 module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig {
   // for documentation of the implementations here, see the QLDoc in the CandidateSig signature module.
-  class Endpoint = ArgumentNode;
+  class Endpoint = ApplicationModeEndpoint;
 
   class EndpointType = AutomodelEndpointTypes::EndpointType;
 
@@ -61,18 +121,18 @@ module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig
   predicate isSanitizer(Endpoint e, EndpointType t) {
     exists(t) and
     (
-      e.getType() instanceof BoxedType
+      e.asNode().getType() instanceof BoxedType
       or
-      e.getType() instanceof PrimitiveType
+      e.asNode().getType() instanceof PrimitiveType
       or
-      e.getType() instanceof NumberType
+      e.asNode().getType() instanceof NumberType
     )
     or
     t instanceof AutomodelEndpointTypes::PathInjectionSinkType and
-    e instanceof PathSanitizer::PathInjectionSanitizer
+    e.asNode() instanceof PathSanitizer::PathInjectionSanitizer
   }
 
-  RelatedLocation asLocation(Endpoint e) { result = e.asExpr() }
+  RelatedLocation asLocation(Endpoint e) { result = e.asTop() }
 
   predicate isKnownKind = AutomodelJavaUtil::isKnownKind/2;
 
@@ -98,16 +158,7 @@ module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig
     ApplicationModeGetCallable::getCallable(e).hasQualifiedName(package, type, name) and
     signature = ExternalFlow::paramsString(ApplicationModeGetCallable::getCallable(e)) and
     ext = "" and
-    (
-      exists(Call c, int argIdx |
-        e.asExpr() = c.getArgument(argIdx) and
-        input = AutomodelJavaUtil::getArgumentForIndex(argIdx)
-      )
-      or
-      exists(Call c |
-        e.asExpr() = c.getQualifier() and input = AutomodelJavaUtil::getArgumentForIndex(-1)
-      )
-    )
+    input = AutomodelJavaUtil::getArgumentForIndex(e.getArgIndex())
   }
 
   /**
@@ -118,7 +169,7 @@ module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig
    */
   RelatedLocation getRelatedLocation(Endpoint e, RelatedLocationType type) {
     type = CallContext() and
-    result = any(Call c | e.asExpr() = [c.getAnArgument(), c.getQualifier()])
+    result = e.asTop()
   }
 }
 
@@ -132,20 +183,15 @@ private module ApplicationModeGetCallable implements AutomodelSharedGetCallable:
   /**
    * Returns the API callable being modeled.
    */
-  Callable getCallable(Endpoint e) {
-    exists(Call c |
-      e.asExpr() = [c.getAnArgument(), c.getQualifier()] and
-      result = c.getCallee()
-    )
-  }
+  Callable getCallable(Endpoint e) { result = e.getCall().getCallee() }
 }
 
 /**
  * Contains endpoints that are defined in QL code rather than as a MaD model. Ideally this predicate
  * should be empty.
  */
 private predicate isCustomSink(Endpoint e, string kind) {
-  e instanceof QueryInjectionSink and kind = "sql"
+  e.asNode() instanceof QueryInjectionSink and kind = "sql"
 }
 
 module CharacteristicsImpl =
@@ -169,14 +215,9 @@ class ApplicationModeMetadataExtractor extends string {
     Endpoint e, string package, string type, string subtypes, string name, string signature,
     string input
   ) {
-    exists(Call call, Callable callable, int argIdx |
-      call.getCallee() = callable and
-      (
-        e.asExpr() = call.getArgument(argIdx)
-        or
-        e.asExpr() = call.getQualifier() and argIdx = -1
-      ) and
-      input = AutomodelJavaUtil::getArgumentForIndex(argIdx) and
+    exists(Callable callable |
+      e.getCall().getCallee() = callable and
+      input = AutomodelJavaUtil::getArgumentForIndex(e.getArgIndex()) and
       package = callable.getDeclaringType().getPackage().getName() and
       // we're using the erased types because the MaD convention is to not specify type parameters.
       // Whether something is or isn't a sink doesn't usually depend on the type parameters.
@@ -253,28 +294,10 @@ private class IsMaDTaintStepCharacteristic extends CharacteristicsImpl::NotASink
   IsMaDTaintStepCharacteristic() { this = "taint step" }
 
   override predicate appliesToEndpoint(Endpoint e) {
-    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(e, _, _) or
-    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(e, _, _) or
-    FlowSummaryImpl::Private::Steps::summaryGetterStep(e, _, _, _) or
-    FlowSummaryImpl::Private::Steps::summarySetterStep(e, _, _, _)
-  }
-}
-
-/**
- * A negative characteristic that filters out qualifiers that are classes (i.e. static calls). These
- * are unlikely to have any non-trivial flow going into them.
- *
- * Technically, an accessed type _could_ come from outside of the source code, but there's not
- * much likelihood of that being user-controlled.
- */
-private class ClassQualifierCharacteristic extends CharacteristicsImpl::NotASinkCharacteristic {
-  ClassQualifierCharacteristic() { this = "class qualifier" }
-
-  override predicate appliesToEndpoint(Endpoint e) {
-    exists(Call c |
-      e.asExpr() = c.getQualifier() and
-      c.getQualifier() instanceof TypeAccess
-    )
+    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(e.asNode(), _, _) or
+    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(e.asNode(), _, _) or
+    FlowSummaryImpl::Private::Steps::summaryGetterStep(e.asNode(), _, _, _) or
+    FlowSummaryImpl::Private::Steps::summarySetterStep(e.asNode(), _, _, _)
   }
 }
 
@@ -351,7 +374,7 @@ private class OtherArgumentToModeledMethodCharacteristic extends Characteristics
 private class FunctionValueCharacteristic extends CharacteristicsImpl::LikelyNotASinkCharacteristic {
   FunctionValueCharacteristic() { this = "function value" }
 
-  override predicate appliesToEndpoint(Endpoint e) { e.asExpr() instanceof FunctionalExpr }
+  override predicate appliesToEndpoint(Endpoint e) { e.asNode().asExpr() instanceof FunctionalExpr }
 }
 
 /**
@@ -371,12 +394,12 @@ private class CannotBeTaintedCharacteristic extends CharacteristicsImpl::LikelyN
    * Holds if the node `n` is known as the predecessor in a modeled flow step.
    */
   private predicate isKnownOutNodeForStep(Endpoint e) {
-    e.asExpr() instanceof Call or // we just assume flow in that case
-    TaintTracking::localTaintStep(_, e) or
-    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(_, e, _) or
-    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(_, e, _) or
-    FlowSummaryImpl::Private::Steps::summaryGetterStep(_, _, e, _) or
-    FlowSummaryImpl::Private::Steps::summarySetterStep(_, _, e, _)
+    e.asNode().asExpr() instanceof Call or // we just assume flow in that case
+    TaintTracking::localTaintStep(_, e.asNode()) or
+    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(_, e.asNode(), _) or
+    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(_, e.asNode(), _) or
+    FlowSummaryImpl::Private::Steps::summaryGetterStep(_, _, e.asNode(), _) or
+    FlowSummaryImpl::Private::Steps::summarySetterStep(_, _, e.asNode(), _)
   }
 }
 

java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql

@@ -33,7 +33,7 @@ private Endpoint getSampleForSignature(
   |
     result =
       rank[n](Endpoint e, Location loc |
-        loc = e.getLocation() and
+        loc = e.asTop().getLocation() and
         meta.hasMetadata(e, package, type, subtypes, name, signature, input)
       |
         e
@@ -75,7 +75,8 @@ where
     |
       sinkType, ", "
     )
-select endpoint, message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
+select endpoint.asNode(),
+  message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()), "CallContext", //
   package, "package", //
   type, "type", //

java/ql/src/Telemetry/AutomodelApplicationModeExtractNegativeExamples.ql

@@ -24,7 +24,7 @@ Endpoint getSampleForCharacteristic(EndpointCharacteristic c, int limit) {
   exists(int n, int num_endpoints | num_endpoints = count(Endpoint e | c.appliesToEndpoint(e)) |
     result =
       rank[n](Endpoint e, Location loc |
-        loc = e.getLocation() and c.appliesToEndpoint(e)
+        loc = e.asTop().getLocation() and c.appliesToEndpoint(e)
       |
         e
         order by
@@ -63,7 +63,8 @@ where
     characteristic2.hasImplications(positiveType, true, confidence2)
   ) and
   message = characteristic
-select endpoint, message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
+select endpoint.asNode(),
+  message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()), "CallContext", //
   package, "package", //
   type, "type", //

java/ql/src/Telemetry/AutomodelApplicationModeExtractPositiveExamples.ql

@@ -22,8 +22,10 @@ where
   not erroneousEndpoints(endpoint, _, _, _, _, false) and
   meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input) and
   // Extract positive examples of sinks belonging to the existing ATM query configurations.
-  CharacteristicsImpl::isKnownSink(endpoint, sinkType)
-select endpoint, sinkType + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
+  CharacteristicsImpl::isKnownSink(endpoint, sinkType) and
+  exists(CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()))
+select endpoint.asNode(),
+  sinkType + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()), "CallContext", //
   package, "package", //
   type, "type", //

java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected

@@ -1,2 +1,3 @@
-| Test.java:16:3:16:11 | reference | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:16:3:16:24 | set(...) | CallContext | file://java.util.concurrent.atomic:1:1:1:1 | java.util.concurrent.atomic | package | file://AtomicReference:1:1:1:1 | AtomicReference | type | file://false:1:1:1:1 | false | subtypes | file://set:1:1:1:1 | set | name | file://(String):1:1:1:1 | (String) | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input |
-| Test.java:21:3:21:10 | supplier | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:21:3:21:16 | get(...) | CallContext | file://java.util.function:1:1:1:1 | java.util.function | package | file://Supplier:1:1:1:1 | Supplier | type | file://true:1:1:1:1 | true | subtypes | file://get:1:1:1:1 | get | name | file://():1:1:1:1 | () | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input |
+| Test.java:16:3:16:11 | reference | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:16:3:16:11 | reference | CallContext | file://java.util.concurrent.atomic:1:1:1:1 | java.util.concurrent.atomic | package | file://AtomicReference:1:1:1:1 | AtomicReference | type | file://false:1:1:1:1 | false | subtypes | file://set:1:1:1:1 | set | name | file://(String):1:1:1:1 | (String) | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input |
+| Test.java:21:3:21:10 | supplier | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:21:3:21:10 | supplier | CallContext | file://java.util.function:1:1:1:1 | java.util.function | package | file://Supplier:1:1:1:1 | Supplier | type | file://true:1:1:1:1 | true | subtypes | file://get:1:1:1:1 | get | name | file://():1:1:1:1 | () | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input |
+| Test.java:51:3:54:3 | new ..[] { .. } | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:51:3:54:3 | walk(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://walk:1:1:1:1 | walk | name | file://(Path,FileVisitOption[]):1:1:1:1 | (Path,FileVisitOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input |

java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.expected

@@ -1,2 +1,2 @@
-| Test.java:40:14:40:21 | openPath | taint step\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:40:4:40:22 | get(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Paths:1:1:1:1 | Paths | type | file://false:1:1:1:1 | false | subtypes | file://get:1:1:1:1 | get | name | file://(String,String[]):1:1:1:1 | (String,String[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input |
-| Test.java:46:4:46:5 | f2 | known non-sink\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:45:10:47:3 | compareTo(...) | CallContext | file://java.io:1:1:1:1 | java.io | package | file://File:1:1:1:1 | File | type | file://true:1:1:1:1 | true | subtypes | file://compareTo:1:1:1:1 | compareTo | name | file://(File):1:1:1:1 | (File) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input |
+| Test.java:46:4:46:5 | f2 | known non-sink\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:46:4:46:5 | f2 | CallContext | file://java.io:1:1:1:1 | java.io | package | file://File:1:1:1:1 | File | type | file://true:1:1:1:1 | true | subtypes | file://compareTo:1:1:1:1 | compareTo | name | file://(File):1:1:1:1 | (File) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input |
+| Test.java:52:4:52:4 | p | taint step\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:52:4:52:4 | p | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://walk:1:1:1:1 | walk | name | file://(Path,FileVisitOption[]):1:1:1:1 | (Path,FileVisitOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input |

java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.expected

@@ -1,3 +1,3 @@
-| Test.java:26:4:26:9 | source | path-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:25:3:29:3 | copy(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input |
-| Test.java:27:4:27:9 | target | path-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:25:3:29:3 | copy(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input |
-| Test.java:34:4:34:11 | openPath | path-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:33:10:35:3 | newInputStream(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://newInputStream:1:1:1:1 | newInputStream | name | file://(Path,OpenOption[]):1:1:1:1 | (Path,OpenOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input |
+| Test.java:26:4:26:9 | source | path-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:26:4:26:9 | source | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input |
+| Test.java:27:4:27:9 | target | path-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:27:4:27:9 | target | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input |
+| Test.java:34:4:34:11 | openPath | path-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:34:4:34:11 | openPath | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://newInputStream:1:1:1:1 | newInputStream | name | file://(Path,OpenOption[]):1:1:1:1 | (Path,OpenOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input |

java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/Test.java

@@ -8,7 +8,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 import java.io.File;
-
+import java.nio.file.FileVisitOption;
 
 class Test {
 	public static void main(String[] args) throws Exception {
@@ -46,5 +46,12 @@ public static int compareFiles(File f1, File f2) {
 			f2 // negative example (modeled as not a sink)
 		);
 	}
+		
+	public static void FilesWalkExample(Path p) throws Exception {
+		Files.walk(
+			p, // negative example (modeled as a taint step)
+			FileVisitOption.FOLLOW_LINKS // the implicit varargs array is a candidate
+		);
+	}
 }