Merge remote-tracking branch 'origin/main' into ruby/weak-cryptographic-algorithm

Author: alexrford

.github/workflows/check-qldoc.yml

@@ -0,0 +1,53 @@
+name: "Check QLdoc coverage"
+
+on:
+  pull_request:
+    paths:
+      - "*/ql/lib/**"
+      - .github/workflows/check-qldoc.yml
+    branches:
+      - main
+      - "rc/*"
+
+jobs:
+  qldoc:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Install CodeQL
+        run: |
+          gh extension install github/gh-codeql
+          gh codeql set-channel nightly
+          gh codeql version
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - name: Check QLdoc coverage
+        shell: bash
+        run: |
+          EXIT_CODE=0
+          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -o '^[a-z]*/ql/lib' || true; } | sort -u)"
+          for pack_dir in ${changed_lib_packs}; do
+            lang="${pack_dir%/ql/lib}"
+            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
+          done
+          git checkout HEAD^
+          for pack_dir in ${changed_lib_packs}; do
+            # When we add a new language, pack_dir would not exist in HEAD^.
+            # In this case the right thing to do is to skip the check.
+            [[ ! -d "${pack_dir}" ]] && continue
+            lang="${pack_dir%/ql/lib}"
+            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
+            awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-current.txt" | sort -u > "${RUNNER_TEMP}/current-undocumented.txt"
+            awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-baseline.txt" | sort -u > "${RUNNER_TEMP}/baseline-undocumented.txt"
+            UNDOCUMENTED="$(grep -f <(comm -13 "${RUNNER_TEMP}/baseline-undocumented.txt" "${RUNNER_TEMP}/current-undocumented.txt") "${RUNNER_TEMP}/${lang}-current.txt" || true)"
+            if [ -n "$UNDOCUMENTED" ]; then
+              echo "$UNDOCUMENTED" | awk -F, '{gsub(/"/,""); print "::warning file='"${pack_dir}"'/"$1",line="$2"::Missing QLdoc for "$5, $3 }'
+              EXIT_CODE=1
+            fi
+          done
+          exit "${EXIT_CODE}"

.github/workflows/ql-for-ql-build.yml

@@ -16,9 +16,10 @@ jobs:
       - uses: actions/checkout@v2
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: javascript # does not matter
+          tools: latest
       - name: Get CodeQL version
         id: get-codeql-version
         run: |
@@ -159,7 +160,7 @@ jobs:
           PACK: ${{ runner.temp }}/pack
       - name: Hack codeql-action options
         run: |
-          JSON=$(jq -nc --arg pack "${PACK}" '.resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
+          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
           echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
         env:
           PACK: ${{ runner.temp }}/pack
@@ -171,22 +172,25 @@ jobs:
           echo "paths:" > ${CONF}
           echo "  - ${FOLDER}" >> ${CONF}
           echo "paths-ignore:" >> ${CONF}
-          echo "  - ql/ql/test" >> ${CONF}
+          echo "  - ql/ql/test" >> ${CONF} 
+          echo "disable-default-queries: true" >> ${CONF}
+          echo "packs:" >> ${CONF}
+          echo "  - codeql/ql" >> ${CONF}
           echo "Config file: "
           cat ${CONF}
         env: 
           CONF: ./ql-for-ql-config.yml
           FOLDER: ${{ matrix.folder }}
-
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: ql
           db-location: ${{ runner.temp }}/db
           config-file: ./ql-for-ql-config.yml
+          tools: latest
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@erik-krogh/ql
+        uses: github/codeql-action/analyze@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with: 
           category: "ql-for-ql-${{ matrix.folder }}"
       - name: Copy sarif file to CWD

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -26,7 +26,7 @@ jobs:
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v2

.github/workflows/ql-for-ql-tests.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v2

CONTRIBUTING.md

@@ -36,7 +36,7 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
     For details, see the [guide on query metadata](docs/query-metadata-style-guide.md).
 
-    Make sure the `select` statement is compatible with the query `@kind`. See [About CodeQL queries](https://help.semmle.com/QL/learn-ql/writing-queries/introduction-to-queries.html#select-clause) on help.semmle.com.
+    Make sure the `select` statement is compatible with the query `@kind`. See [About CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/about-codeql-queries/#select-clause) on codeql.github.com.
 
 3. **Formatting**
 

config/identical-files.json

@@ -27,7 +27,8 @@
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForLibraries.qll"
   ],
   "DataFlow Java/C++/C#/Python Common": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
@@ -54,7 +55,8 @@
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking2/TaintTrackingImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking3/TaintTrackingImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking4/TaintTrackingImpl.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttrackingforlibraries/TaintTrackingImpl.qll"
   ],
   "DataFlow Java/C++/C#/Python Consistency checks": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll",
@@ -73,6 +75,10 @@
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll"
   ],
+  "Model as Data Generation Java/C# - CaptureModels": [
+    "java/ql/src/utils/model-generator/internal/CaptureModels.qll",
+    "csharp/ql/src/utils/model-generator/internal/CaptureModels.qll"
+  ],
   "Sign Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/Sign.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/Sign.qll"
@@ -472,11 +478,12 @@
     "python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll",
     "ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll"
   ],
-  "ReDoS Exponential Python/JS": [
+  "ReDoS Exponential Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll",
-    "python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll"
+    "python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll"
   ],
-  "ReDoS Polynomial Python/JS": [
+  "ReDoS Polynomial Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll",
     "python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll",
     "ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll"
@@ -507,5 +514,39 @@
     "java/ql/lib/semmle/code/java/dataflow/internal/AccessPathSyntax.qll",
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll"
+  ],
+  "IncompleteUrlSubstringSanitization": [
+    "javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.qll",
+    "ruby/ql/src/queries/security/cwe-020/IncompleteUrlSubstringSanitization.qll"
+  ],
+  "Concepts Python/Ruby/JS": [
+    "python/ql/lib/semmle/python/internal/ConceptsShared.qll",
+    "ruby/ql/lib/codeql/ruby/internal/ConceptsShared.qll",
+    "javascript/ql/lib/semmle/javascript/internal/ConceptsShared.qll"
+  ],
+  "Hostname Regexp queries": [
+    "javascript/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
+    "python/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
+    "ruby/ql/src/queries/security/cwe-020/HostnameRegexpShared.qll"
+  ],
+  "ApiGraphModels": [
+    "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll",
+    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll"
+  ],
+  "TaintedFormatStringQuery Ruby/JS": [
+    "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/TaintedFormatStringQuery.qll"
+  ],
+  "TaintedFormatStringCustomizations Ruby/JS": [
+    "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringCustomizations.qll",
+    "ruby/ql/lib/codeql/ruby/security/TaintedFormatStringCustomizations.qll"
+  ],
+  "HttpToFileAccessQuery JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/dataflow/HttpToFileAccessQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/HttpToFileAccessQuery.qll"
+  ],
+  "HttpToFileAccessCustomizations JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/dataflow/HttpToFileAccessCustomizations.qll",
+    "ruby/ql/lib/codeql/ruby/security/HttpToFileAccessCustomizations.qll"
   ]
-}
+}

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.0.11
+
+### Minor Analysis Improvements
+
+* Many queries now support structured bindings, as structured bindings are now handled in the IR translation.
+
 ## 0.0.10
 
 ### New Features
@@ -6,6 +12,7 @@
 
 ## 0.0.9
 
+
 ## 0.0.8
 
 ### Deprecated APIs

cpp/ql/lib/DefaultOptions.qll

@@ -54,11 +54,13 @@ class Options extends string {
    *
    * By default, this holds for `exit`, `_exit`, `abort`, `__assert_fail`,
    * `longjmp`, `__builtin_unreachable` and any function with a
-   * `noreturn` attribute.
+   * `noreturn` attribute or specifier.
    */
   predicate exits(Function f) {
     f.getAnAttribute().hasName("noreturn")
     or
+    f.getASpecifier().hasName("noreturn")
+    or
     f.hasGlobalOrStdName([
         "exit", "_exit", "abort", "__assert_fail", "longjmp", "__builtin_unreachable"
       ])

cpp/ql/lib/Options.qll

@@ -39,7 +39,7 @@ class CustomOptions extends Options {
    *
    * By default, this holds for `exit`, `_exit`, `abort`, `__assert_fail`,
    * `longjmp`, `error`, `__builtin_unreachable` and any function with a
-   * `noreturn` attribute.
+   * `noreturn` attribute or specifier.
    */
   override predicate exits(Function f) { Options.super.exits(f) }
 

cpp/ql/lib/change-notes/2022-02-07-deprecated-acronyms.md

@@ -0,0 +1,5 @@
+---
+category: deprecated
+---
+* Many classes/predicates/modules that had upper-case acronyms have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.