Merge branch 'main' into stdlib-http-source-modeling

Author: RasmusWL

.github/workflows/generate-query-help-docs.yml

@@ -0,0 +1,60 @@
+name: Generate CodeQL query help documentation using Sphinx
+
+on:
+  workflow_dispatch:
+      inputs:
+        description:
+          description: A description of the purpose of this job. For human consumption.
+          required: false
+  push:
+    branches:
+     - 'lgtm.com'
+  pull_request:
+    paths:
+      - '.github/workflows/generate-query-help-docs.yml'
+      - 'docs/codeql/query-help/**'
+      
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone github/codeql
+      uses: actions/checkout@v2
+      with:
+        path: codeql
+    - name: Clone github/codeql-go
+      uses: actions/checkout@v2
+      with:
+        repository: 'github/codeql-go'
+        path: codeql-go
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Download CodeQL CLI
+      uses: dsaltares/fetch-gh-release-asset@aa37ae5c44d3c9820bc12fe675e8670ecd93bd1c
+      with:
+        repo: "github/codeql-cli-binaries"
+        version: "latest"
+        file: "codeql-linux64.zip"
+        token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Unzip CodeQL CLI
+      run: unzip -d codeql-cli codeql-linux64.zip
+    - name: Set up query help docs folder  
+      run: |
+        cp -r codeql/docs/codeql/** .
+    - name: Query help to markdown
+      run: |
+        PATH="$PATH:codeql-cli/codeql" python codeql/docs/codeql/query-help-markdown.py
+    - name: Run Sphinx for query help
+      uses: ammaraskar/sphinx-action@8b4f60114d7fd1faeba1a712269168508d4750d2 # v0.4
+      with:
+        docs-folder: "query-help/"
+        pre-build-command: "python -m pip install --upgrade recommonmark"
+        build-command: "sphinx-build -b dirhtml . _build" 
+    - name: Upload HTML artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: query-help-html
+        path: query-help/_build
+    

change-notes/1.26/analysis-python.md

@@ -4,19 +4,34 @@ The following changes in version 1.26 affect Python analysis in all applications
 
 ## General improvements
 
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-
-
 ## Changes to existing queries
 
 | **Query**                  | **Expected impact**    | **Change**                                                       |
 |----------------------------|------------------------|------------------------------------------------------------------|
-
-
+|`py/unsafe-deserialization` | Different results. | The underlying data flow library has been changed. See below for more details. |
+|`py/path-injection` | Different results. | The underlying data flow library has been changed. See below for more details. |
+|`py/command-line-injection` | Different results. | The underlying data flow library has been changed. See below for more details. |
+|`py/reflective-xss` | Different results. | The underlying data flow library has been changed. See below for more details. |
+|`py/sql-injection` | Different results. | The underlying data flow library has been changed. See below for more details. |
+|`py/code-injection` | Different results. | The underlying data flow library has been changed. See below for more details. |
 ## Changes to libraries
-
+* Some of the security queries now use the shared data flow library for data flow and taint tracking. This has resulted in an overall more robust and accurate analysis. The libraries mentioned below have been modelled in this new framework. Other libraries (e.g. the web framework `CherryPy`) have not been modelled yet, and this may lead to a temporary loss of results for these frameworks.
+* Improved modelling of the following serialization libraries:
+  - `PyYAML`
+  - `dill`
+  - `pickle`
+  - `marshal`
+* Improved modelling of the following web frameworks:
+  - `Django` (Note that modelling of class-based response handlers is currently incomplete.)
+  - `Flask`
+* Support for Werkzeug `MultiDict`.
+* Support for the [Python Database API Specification v2.0 (PEP-249)](https://www.python.org/dev/peps/pep-0249/), including the following libraries:
+  - `MySQLdb`
+  - `mysql-connector-python`
+  - `django.db`
+* Improved modelling of the following command execution libraries:
+  - `Fabric`
+  - `Invoke`
+* Improved modelling of security-related standard library modules, such as `os`, `popen2`, `platform`, and `base64`.
+* The original versions of the updated queries have been preserved [here](https://github.com/github/codeql/tree/main/python/ql/src/experimental/Security-old-dataflow).
 * Added taint tracking support for string formatting through f-strings.

cpp/ql/src/Best Practices/Magic Constants/MagicConstants.qll

@@ -8,168 +8,41 @@ import semmle.code.cpp.AutogeneratedFile
 predicate trivialPositiveIntValue(string s) {
   // Small numbers
   s = [0 .. 20].toString() or
-  // Popular powers of two (decimal)
-  s = "16" or
-  s = "24" or
-  s = "32" or
-  s = "64" or
-  s = "128" or
-  s = "256" or
-  s = "512" or
-  s = "1024" or
-  s = "2048" or
-  s = "4096" or
-  s = "16384" or
-  s = "32768" or
-  s = "65536" or
-  s = "1048576" or
-  s = "2147483648" or
-  s = "4294967296" or
-  // Popular powers of two, minus one (decimal)
-  s = "15" or
-  s = "31" or
-  s = "63" or
-  s = "127" or
-  s = "255" or
-  s = "511" or
-  s = "1023" or
-  s = "2047" or
-  s = "4095" or
-  s = "16383" or
-  s = "32767" or
-  s = "65535" or
-  s = "1048577" or
-  s = "2147483647" or
-  s = "4294967295" or
-  // Popular powers of two (32-bit hex)
-  s = "0x00000001" or
-  s = "0x00000002" or
-  s = "0x00000004" or
-  s = "0x00000008" or
-  s = "0x00000010" or
-  s = "0x00000020" or
-  s = "0x00000040" or
-  s = "0x00000080" or
-  s = "0x00000100" or
-  s = "0x00000200" or
-  s = "0x00000400" or
-  s = "0x00000800" or
-  s = "0x00001000" or
-  s = "0x00002000" or
-  s = "0x00004000" or
-  s = "0x00008000" or
-  s = "0x00010000" or
-  s = "0x00020000" or
-  s = "0x00040000" or
-  s = "0x00080000" or
-  s = "0x00100000" or
-  s = "0x00200000" or
-  s = "0x00400000" or
-  s = "0x00800000" or
-  s = "0x01000000" or
-  s = "0x02000000" or
-  s = "0x04000000" or
-  s = "0x08000000" or
-  s = "0x10000000" or
-  s = "0x20000000" or
-  s = "0x40000000" or
-  s = "0x80000000" or
-  // Popular powers of two, minus one (32-bit hex)
-  s = "0x00000001" or
-  s = "0x00000003" or
-  s = "0x00000007" or
-  s = "0x0000000f" or
-  s = "0x0000001f" or
-  s = "0x0000003f" or
-  s = "0x0000007f" or
-  s = "0x000000ff" or
-  s = "0x000001ff" or
-  s = "0x000003ff" or
-  s = "0x000007ff" or
-  s = "0x00000fff" or
-  s = "0x00001fff" or
-  s = "0x00003fff" or
-  s = "0x00007fff" or
-  s = "0x0000ffff" or
-  s = "0x0001ffff" or
-  s = "0x0003ffff" or
-  s = "0x0007ffff" or
-  s = "0x000fffff" or
-  s = "0x001fffff" or
-  s = "0x003fffff" or
-  s = "0x007fffff" or
-  s = "0x00ffffff" or
-  s = "0x01ffffff" or
-  s = "0x03ffffff" or
-  s = "0x07ffffff" or
-  s = "0x0fffffff" or
-  s = "0x1fffffff" or
-  s = "0x3fffffff" or
-  s = "0x7fffffff" or
-  s = "0xffffffff" or
-  // Popular powers of two (16-bit hex)
-  s = "0x0001" or
-  s = "0x0002" or
-  s = "0x0004" or
-  s = "0x0008" or
-  s = "0x0010" or
-  s = "0x0020" or
-  s = "0x0040" or
-  s = "0x0080" or
-  s = "0x0100" or
-  s = "0x0200" or
-  s = "0x0400" or
-  s = "0x0800" or
-  s = "0x1000" or
-  s = "0x2000" or
-  s = "0x4000" or
-  s = "0x8000" or
-  // Popular powers of two, minus one (16-bit hex)
-  s = "0x0001" or
-  s = "0x0003" or
-  s = "0x0007" or
-  s = "0x000f" or
-  s = "0x001f" or
-  s = "0x003f" or
-  s = "0x007f" or
-  s = "0x00ff" or
-  s = "0x01ff" or
-  s = "0x03ff" or
-  s = "0x07ff" or
-  s = "0x0fff" or
-  s = "0x1fff" or
-  s = "0x3fff" or
-  s = "0x7fff" or
-  s = "0xffff" or
-  // Popular powers of two (8-bit hex)
-  s = "0x01" or
-  s = "0x02" or
-  s = "0x04" or
-  s = "0x08" or
-  s = "0x10" or
-  s = "0x20" or
-  s = "0x40" or
-  s = "0x80" or
-  // Popular powers of two, minus one (8-bit hex)
-  s = "0x01" or
-  s = "0x03" or
-  s = "0x07" or
-  s = "0x0f" or
-  s = "0x1f" or
-  s = "0x3f" or
-  s = "0x7f" or
-  s = "0xff" or
-  s = "0x00" or
-  // Powers of ten
-  s = "10" or
-  s = "100" or
-  s = "1000" or
-  s = "10000" or
-  s = "100000" or
-  s = "1000000" or
-  s = "10000000" or
-  s = "100000000" or
-  s = "1000000000"
+  s =
+    [
+      // Popular powers of two (decimal)
+      "16", "24", "32", "64", "128", "256", "512", "1024", "2048", "4096", "16384", "32768",
+      "65536", "1048576", "2147483648", "4294967296",
+      // Popular powers of two, minus one (decimal)
+      "15", "31", "63", "127", "255", "511", "1023", "2047", "4095", "16383", "32767", "65535",
+      "1048577", "2147483647", "4294967295",
+      // Popular powers of two (32-bit hex)
+      "0x00000001", "0x00000002", "0x00000004", "0x00000008", "0x00000010", "0x00000020",
+      "0x00000040", "0x00000080", "0x00000100", "0x00000200", "0x00000400", "0x00000800",
+      "0x00001000", "0x00002000", "0x00004000", "0x00008000", "0x00010000", "0x00020000",
+      "0x00040000", "0x00080000", "0x00100000", "0x00200000", "0x00400000", "0x00800000",
+      "0x01000000", "0x02000000", "0x04000000", "0x08000000", "0x10000000", "0x20000000",
+      "0x40000000", "0x80000000",
+      // Popular powers of two, minus one (32-bit hex)
+      "0x00000001", "0x00000003", "0x00000007", "0x0000000f", "0x0000001f", "0x0000003f",
+      "0x0000007f", "0x000000ff", "0x000001ff", "0x000003ff", "0x000007ff", "0x00000fff",
+      "0x00001fff", "0x00003fff", "0x00007fff", "0x0000ffff", "0x0001ffff", "0x0003ffff",
+      "0x0007ffff", "0x000fffff", "0x001fffff", "0x003fffff", "0x007fffff", "0x00ffffff",
+      "0x01ffffff", "0x03ffffff", "0x07ffffff", "0x0fffffff", "0x1fffffff", "0x3fffffff",
+      "0x7fffffff", "0xffffffff",
+      // Popular powers of two (16-bit hex)
+      "0x0001", "0x0002", "0x0004", "0x0008", "0x0010", "0x0020", "0x0040", "0x0080", "0x0100",
+      "0x0200", "0x0400", "0x0800", "0x1000", "0x2000", "0x4000", "0x8000",
+      // Popular powers of two, minus one (16-bit hex)
+      "0x0001", "0x0003", "0x0007", "0x000f", "0x001f", "0x003f", "0x007f", "0x00ff", "0x01ff",
+      "0x03ff", "0x07ff", "0x0fff", "0x1fff", "0x3fff", "0x7fff", "0xffff",
+      // Popular powers of two (8-bit hex)
+      "0x01", "0x02", "0x04", "0x08", "0x10", "0x20", "0x40", "0x80",
+      // Popular powers of two, minus one (8-bit hex)
+      "0x01", "0x03", "0x07", "0x0f", "0x1f", "0x3f", "0x7f", "0xff", "0x00",
+      // Powers of ten
+      "10", "100", "1000", "10000", "100000", "1000000", "10000000", "100000000", "1000000000"
+    ]
 }
 
 predicate trivialIntValue(string s) {
@@ -235,10 +108,7 @@ predicate joiningStringTrivial(Literal lit) {
   // understand (which is against the spirit of these queries).
   stringLiteral(lit) and
   exists(FunctionCall fc |
-    (
-      fc.getTarget().getName() = "operator+" or
-      fc.getTarget().getName() = "operator<<"
-    ) and
+    fc.getTarget().getName() = ["operator+", "operator<<"] and
     fc.getAnArgument().getAChild*() = lit
   ) and
   lit.getValue().length() < 16

cpp/ql/src/DefaultOptions.qll

@@ -59,14 +59,9 @@ class Options extends string {
   predicate exits(Function f) {
     f.getAnAttribute().hasName("noreturn")
     or
-    exists(string name | f.hasGlobalOrStdName(name) |
-      name = "exit" or
-      name = "_exit" or
-      name = "abort" or
-      name = "__assert_fail" or
-      name = "longjmp" or
-      name = "__builtin_unreachable"
-    )
+    f.hasGlobalOrStdName([
+        "exit", "_exit", "abort", "__assert_fail", "longjmp", "__builtin_unreachable"
+      ])
     or
     CustomOptions::exits(f) // old Options.qll
   }

cpp/ql/src/JPL_C/LOC-2/Rule 05/HeapMemory.ql

@@ -21,15 +21,7 @@ class Initialization extends Function {
 }
 
 class Allocation extends FunctionCall {
-  Allocation() {
-    exists(string name | name = this.getTarget().getName() |
-      name = "malloc" or
-      name = "calloc" or
-      name = "alloca" or
-      name = "sbrk" or
-      name = "valloc"
-    )
-  }
+  Allocation() { this.getTarget().getName() = ["malloc", "calloc", "alloca", "sbrk", "valloc"] }
 }
 
 from Function f, Allocation a

cpp/ql/src/JPL_C/LOC-2/Rule 07/ThreadSafety.ql

@@ -13,13 +13,8 @@ import cpp
 
 class ForbiddenCall extends FunctionCall {
   ForbiddenCall() {
-    exists(string name | name = this.getTarget().getName() |
-      name = "task_delay" or
-      name = "taskDelay" or
-      name = "sleep" or
-      name = "nanosleep" or
-      name = "clock_nanosleep"
-    )
+    this.getTarget().getName() =
+      ["task_delay", "taskDelay", "sleep", "nanosleep", "clock_nanosleep"]
   }
 }