Shared: Post-processing query for inline test expectations

hvitved · hvitved · commit e7a3e6bfedb6 · 2024-10-29T13:35:29.000+01:00
diff --git a/shared/util/codeql/util/test/InlineExpectationsTest.qll b/shared/util/codeql/util/test/InlineExpectationsTest.qll
@@ -134,8 +134,31 @@ module Make<InlineExpectationsTestSig Impl> {
      * predicate for an active test will be ignored. This makes it possible to write multiple tests in
      * different `.ql` files that all query the same source code.
      */
+    bindingset[result]
     string getARelevantTag();
 
+    /**
+     * Holds if expected tag `expectedTag` matches actual tag `actualTag`.
+     *
+     * This is normally defined as `expectedTag = actualTag`.
+     */
+    bindingset[expectedTag, actualTag]
+    default predicate tagMatches(string expectedTag, string actualTag) { expectedTag = actualTag }
+
+    /** Holds if expectations marked with `expectedTag` are optional. */
+    bindingset[expectedTag]
+    default predicate tagIsOptional(string expectedTag) { none() }
+
+    /**
+     * Holds if expected value `expectedValue` matches actual value `actualValue`.
+     *
+     * This is normally defined as `expectedValue = actualValue`.
+     */
+    bindingset[expectedValue, actualValue]
+    default predicate valueMatches(string expectedValue, string actualValue) {
+      expectedValue = actualValue
+    }
+
     /**
      * Returns the actual results of the query that is being tested. Each result consist of the
      * following values:
@@ -200,13 +223,13 @@ module Make<InlineExpectationsTestSig Impl> {
         not exists(ActualTestResult actualResult | expectation.matchesActualResult(actualResult)) and
         expectation.getTag() = TestImpl::getARelevantTag() and
         element = expectation and
-        (
-          expectation instanceof GoodTestExpectation and
-          message = "Missing result: " + expectation.getExpectationText()
-          or
-          expectation instanceof FalsePositiveTestExpectation and
-          message = "Fixed spurious result: " + expectation.getExpectationText()
-        )
+        not expectation.isOptional()
+      |
+        expectation instanceof GoodTestExpectation and
+        message = "Missing result: " + expectation.getExpectationText()
+        or
+        expectation instanceof FalsePositiveTestExpectation and
+        message = "Fixed spurious result: " + expectation.getExpectationText()
       )
       or
       exists(InvalidTestExpectation expectation |
@@ -311,9 +334,11 @@ module Make<InlineExpectationsTestSig Impl> {
 
       predicate matchesActualResult(ActualTestResult actualResult) {
         onSameLine(pragma[only_bind_into](this), actualResult) and
-        this.getTag() = actualResult.getTag() and
-        this.getValue() = actualResult.getValue()
+        TestImpl::tagMatches(this.getTag(), actualResult.getTag()) and
+        TestImpl::valueMatches(this.getValue(), actualResult.getValue())
       }
+
+      predicate isOptional() { TestImpl::tagIsOptional(tag) }
     }
 
     // Note: These next three classes correspond to all the possible values of type `TColumn`.
@@ -337,6 +362,18 @@ module Make<InlineExpectationsTestSig Impl> {
       string getExpectation() { result = expectation }
     }
 
+    /**
+     * Gets a test expectation that matches the actual result at the given location.
+     */
+    ValidTestExpectation getAMatchingExpectation(
+      Impl::Location location, string element, string tag, string val, boolean optional
+    ) {
+      exists(ActualTestResult actualResult |
+        result.matchesActualResult(actualResult) and
+        actualResult = TActualResult(location, element, tag, val, optional)
+      )
+    }
+
     query predicate testFailures(FailureLocatable element, string message) {
       hasFailureMessage(element, message)
     }
@@ -385,6 +422,7 @@ module Make<InlineExpectationsTestSig Impl> {
    * ```
    */
   module MergeTests<TestSig TestImpl1, TestSig TestImpl2> implements TestSig {
+    bindingset[result]
     string getARelevantTag() {
       result = TestImpl1::getARelevantTag() or result = TestImpl2::getARelevantTag()
     }
@@ -408,6 +446,7 @@ module Make<InlineExpectationsTestSig Impl> {
   module MergeTests3<TestSig TestImpl1, TestSig TestImpl2, TestSig TestImpl3> implements TestSig {
     private module M = MergeTests<MergeTests<TestImpl1, TestImpl2>, TestImpl3>;
 
+    bindingset[result]
     string getARelevantTag() { result = M::getARelevantTag() }
 
     predicate hasActualResult(Impl::Location location, string element, string tag, string value) {
@@ -427,6 +466,7 @@ module Make<InlineExpectationsTestSig Impl> {
   {
     private module M = MergeTests<MergeTests3<TestImpl1, TestImpl2, TestImpl3>, TestImpl4>;
 
+    bindingset[result]
     string getARelevantTag() { result = M::getARelevantTag() }
 
     predicate hasActualResult(Impl::Location location, string element, string tag, string value) {
@@ -448,6 +488,7 @@ module Make<InlineExpectationsTestSig Impl> {
     private module M =
       MergeTests<MergeTests4<TestImpl1, TestImpl2, TestImpl3, TestImpl4>, TestImpl5>;
 
+    bindingset[result]
     string getARelevantTag() { result = M::getARelevantTag() }
 
     predicate hasActualResult(Impl::Location location, string element, string tag, string value) {
@@ -590,3 +631,262 @@ private string expectationPattern() {
     result = tags + "(?:=" + value + ")?"
   )
 }
+
+/**
+ * Provides logic for creating a `@kind test-postprocess` query that checks
+ * inline test expectations using `$ Alert` markers.
+ *
+ * The postprocessing query works for queries of kind `problem` and `path-problem`,
+ * and each query result must have a matching `$ Alert` comment. It is possible to
+ * augment the comment with a query ID, in order to support cases where multiple
+ * `.qlref` tests share the same test code:
+ *
+ * ```rust
+ * var x = ""; // $ Alert[rust/unused-value]
+ * return;
+ * foo();      // $ Alert[rust/unreachable-code]
+ * ```
+ *
+ * In the example above, the `$ Alert[rust/unused-value]` commment is only taken
+ * into account in the test for the query with ID `rust/unused-value`, and vice
+ * versa for the `$ Alert[rust/unreachable-code]` comment.
+ *
+ * For `path-problem` queries, each source and sink must additionally be annotated
+ * (`$ Source` and `$ Sink`, respectively), except when their location coincides
+ * with the location of the alert itself, in which case only `$ Alert` is needed.
+ *
+ * Example:
+ *
+ * ```csharp
+ * var queryParam = Request.QueryString["param"]; // $ Source
+ * Write(Html.Raw(queryParam));                   // $ Alert
+ * ```
+ *
+ * Morover, it is possible to tag sources with a unique identifier:
+ *
+ * ```csharp
+ * var queryParam = Request.QueryString["param"]; // $ Source=source1
+ * Write(Html.Raw(queryParam));                   // $ Alert=source1
+ * ```
+ *
+ * In this case, the source and sink must have the same tag in order
+ * to be matched.
+ */
+module TestPostProcessing {
+  external predicate queryResults(string relation, int row, int column, string data);
+
+  external predicate queryRelations(string relation);
+
+  external predicate queryMetadata(string key, string value);
+
+  private string getQueryId() { queryMetadata("id", result) }
+
+  private string getQueryKind() { queryMetadata("kind", result) }
+
+  signature module InputSig<InlineExpectationsTestSig Input> {
+    string getRelativeUrl(Input::Location location);
+  }
+
+  module Make<InlineExpectationsTestSig Input, InputSig<Input> Input2> {
+    private import InlineExpectationsTest as InlineExpectationsTest
+    private import InlineExpectationsTest::Make<Input>
+
+    /**
+     * Gets the tag to be used for the path-problem source at result row `row`.
+     *
+     * This is either `Source` or `Alert`, depending on whether the location
+     * of the source matches the location of the alert.
+     */
+    private string getSourceTag(int row) {
+      getQueryKind() = "path-problem" and
+      exists(string loc | queryResults("#select", row, 2, loc) |
+        if queryResults("#select", row, 0, loc) then result = "Alert" else result = "Source"
+      )
+    }
+
+    /**
+     * Gets the tag to be used for the path-problem sink at result row `row`.
+     *
+     * This is either `Sink` or `Alert`, depending on whether the location
+     * of the sink matches the location of the alert.
+     */
+    private string getSinkTag(int row) {
+      getQueryKind() = "path-problem" and
+      exists(string loc | queryResults("#select", row, 4, loc) |
+        if queryResults("#select", row, 0, loc) then result = "Alert" else result = "Sink"
+      )
+    }
+
+    /**
+     * A configuration for matching `// $ Source=foo` comments against actual
+     * path-problem sources.
+     */
+    private module PathProblemSourceTestInput implements TestSig {
+      string getARelevantTag() { result = getSourceTag(_) }
+
+      bindingset[expectedValue, actualValue]
+      predicate valueMatches(string expectedValue, string actualValue) {
+        exists(expectedValue) and
+        actualValue = ""
+      }
+
+      additional predicate hasPathProblemSource(
+        int row, Input::Location location, string element, string tag, string value
+      ) {
+        getQueryKind() = "path-problem" and
+        exists(string loc |
+          queryResults("#select", row, 2, loc) and
+          queryResults("#select", row, 3, element) and
+          tag = getSourceTag(row) and
+          value = "" and
+          Input2::getRelativeUrl(location) = loc
+        )
+      }
+
+      predicate hasActualResult(Input::Location location, string element, string tag, string value) {
+        hasPathProblemSource(_, location, element, tag, value)
+      }
+    }
+
+    private module PathProblemSourceTest = MakeTest<PathProblemSourceTestInput>;
+
+    private module TestInput implements TestSig {
+      bindingset[result]
+      string getARelevantTag() { any() }
+
+      private string getTagRegex() {
+        exists(string sourceSinkTags |
+          getQueryKind() = "problem" and
+          sourceSinkTags = ""
+          or
+          sourceSinkTags = "|" + getSourceTag(_) + "|" + getSinkTag(_)
+        |
+          result = "(Alert" + sourceSinkTags + ")(\\[(.*)\\])?"
+        )
+      }
+
+      bindingset[expectedTag, actualTag]
+      predicate tagMatches(string expectedTag, string actualTag) {
+        actualTag = expectedTag.regexpCapture(getTagRegex(), 1) and
+        (
+          // expected tag is annotated with a query ID
+          getQueryId() = expectedTag.regexpCapture(getTagRegex(), 3)
+          or
+          // expected tag is not annotated with a query ID
+          not exists(expectedTag.regexpCapture(getTagRegex(), 3))
+        )
+      }
+
+      bindingset[expectedTag]
+      predicate tagIsOptional(string expectedTag) {
+        // ignore irrelevant tags
+        not expectedTag.regexpMatch(getTagRegex())
+        or
+        // ignore tags annotated with a query ID that does not match the current query ID
+        exists(string queryId |
+          queryId = expectedTag.regexpCapture(getTagRegex(), 3) and
+          queryId != getQueryId()
+        )
+      }
+
+      bindingset[expectedValue, actualValue]
+      predicate valueMatches(string expectedValue, string actualValue) {
+        expectedValue = actualValue
+        or
+        actualValue = ""
+      }
+
+      private predicate hasPathProblemSource = PathProblemSourceTestInput::hasPathProblemSource/5;
+
+      /**
+       * Gets the expected sink value for result row `row`. This value must
+       * match the value at the corresponding path-problem source (if it is
+       * present).
+       */
+      private string getSinkValue(int row) {
+        exists(Input::Location location, string element, string tag, string val |
+          hasPathProblemSource(row, location, element, tag, val) and
+          result =
+            PathProblemSourceTest::getAMatchingExpectation(location, element, tag, val, false)
+                .getValue()
+        )
+      }
+
+      private predicate hasPathProblemSink(
+        int row, Input::Location location, string element, string tag, string value
+      ) {
+        getQueryKind() = "path-problem" and
+        exists(string loc |
+          queryResults("#select", row, 4, loc) and
+          queryResults("#select", row, 5, element) and
+          tag = getSinkTag(row) and
+          Input2::getRelativeUrl(location) = loc
+        |
+          not exists(getSinkValue(row)) and value = ""
+          or
+          value = getSinkValue(row)
+        )
+      }
+
+      private predicate hasAlert(Input::Location location, string element, string tag, string value) {
+        getQueryKind() = ["problem", "path-problem"] and
+        exists(int row, string loc |
+          queryResults("#select", row, 0, loc) and
+          queryResults("#select", row, 2, element) and
+          tag = "Alert" and
+          value = "" and
+          Input2::getRelativeUrl(location) = loc and
+          not hasPathProblemSource(row, location, _, _, _) and
+          not hasPathProblemSink(row, location, _, _, _)
+        )
+      }
+
+      predicate hasActualResult(Input::Location location, string element, string tag, string value) {
+        hasPathProblemSource(_, location, element, tag, value)
+        or
+        hasPathProblemSink(_, location, element, tag, value)
+        or
+        hasAlert(location, element, tag, value)
+      }
+    }
+
+    private module Test = MakeTest<TestInput>;
+
+    private newtype TTestFailure =
+      MkTestFailure(Test::FailureLocatable f, string message) { Test::testFailures(f, message) }
+
+    private predicate rankedTestFailures(int i, MkTestFailure f) {
+      f =
+        rank[i](MkTestFailure f0, Test::FailureLocatable fl, string message, string filename,
+          int startLine, int startColumn, int endLine, int endColumn |
+          f0 = MkTestFailure(fl, message) and
+          fl.getLocation().hasLocationInfo(filename, startLine, startColumn, endLine, endColumn)
+        |
+          f0 order by filename, startLine, startColumn, endLine, endColumn, message
+        )
+    }
+
+    query predicate results(string relation, int row, int column, string data) {
+      queryResults(relation, row, column, data)
+      or
+      exists(MkTestFailure f, Test::FailureLocatable fl, string message |
+        relation = "testFailures" and
+        rankedTestFailures(row, f) and
+        f = MkTestFailure(fl, message)
+      |
+        column = 0 and data = Input2::getRelativeUrl(fl.getLocation())
+        or
+        column = 1 and data = fl.toString()
+        or
+        column = 2 and data = message
+      )
+    }
+
+    query predicate resultRelations(string relation) {
+      queryRelations(relation)
+      or
+      Test::testFailures(_, _) and
+      relation = "testFailures"
+    }
+  }
+}
