Python: CG trace: Add helper.sh to run tracing against real projects

Author: RasmusWL

python/tools/recorded-call-graph-metrics/.gitignore

@@ -8,4 +8,6 @@ tests/cg-trace-test-db
 # Artifact from building `pip install -e .`
 src/cg_trace.egg-info/
 
+projects/
+
 venv/

python/tools/recorded-call-graph-metrics/helper.sh

@@ -0,0 +1,159 @@
+#!/bin/bash
+
+set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+PROJECTS_FILE="$SCRIPTDIR/projects.json"
+
+PROJECT_BASE_DIR="$SCRIPTDIR/projects"
+
+repo_dir() {
+    echo "$PROJECT_BASE_DIR/$1/repo"
+}
+
+venv_dir() {
+    echo "$PROJECT_BASE_DIR/$1/venv"
+}
+
+trace_dir() {
+    echo "$PROJECT_BASE_DIR/$1/traces"
+}
+
+db_path() {
+    echo "$PROJECT_BASE_DIR/$1/$1-db"
+}
+
+help() {
+    echo -n """\
+$0 help                 This message
+$0 projects             List projects
+$0 repo <projects>      Fetch repo for projects
+$0 setup <projects>     Perform setup steps for projects (install dependencies)
+$0 trace <projects>     Trace projects
+$0 db <projects>        Build databases for projects
+$0 all <projects>       Perform all the above steps for projects
+"""
+}
+
+projects() {
+    jq -r 'keys[]' "$PROJECTS_FILE"
+}
+
+check_project_exists() {
+    if ! jq -e ".\"$1\"" "$PROJECTS_FILE" &>/dev/null; then
+        echo "ERROR: '$1' not a known project, see '$0 projects'"
+        exit 1
+    fi
+}
+
+repo() {
+    for project in $@; do
+        check_project_exists $project
+
+        echo "Cloning repo for $project"
+
+        REPO_DIR=$(repo_dir $project)
+
+        if [[ -d "$REPO_DIR" ]]; then
+            echo "Repo already cloned in $REPO_DIR"
+            continue;
+        fi
+
+        REPO_URL=$(jq -e -r ".\"$project\".repo" "$PROJECTS_FILE")
+        SHA=$(jq -e -r ".\"$project\".sha" "$PROJECTS_FILE")
+
+        mkdir -p "$REPO_DIR"
+        cd "$REPO_DIR"
+        git init
+        git remote add origin $REPO_URL
+        git fetch --depth 1 origin $SHA
+        git -c advice.detachedHead=False checkout FETCH_HEAD
+    done
+}
+
+setup() {
+    for project in $@; do
+        check_project_exists $project
+
+        echo "Setting up $project"
+
+        python3 -m venv $(venv_dir $project)
+        source $(venv_dir $project)/bin/activate
+
+        pip install -e "$SCRIPTDIR"
+
+        IFS=$'\n'
+        setup_commands=($(jq -r ".\"$project\".setup[]" $PROJECTS_FILE))
+        unset IFS
+        for setup_command in "${setup_commands[@]}"; do
+            echo "Running '$setup_command'"
+            $setup_command
+        done
+
+        # deactivate venv again
+        deactivate
+    done
+}
+
+trace() {
+    for project in $@; do
+        check_project_exists $project
+
+        echo "Tracing '$project"
+
+        source $(venv_dir $project)/bin/activate
+
+        REPO_DIR=$(repo_dir $project)
+        cd "$REPO_DIR"
+
+        mkdir -p $(trace_dir $project)
+
+        MODULE_COMMAND=$(jq -r ".\"$project\".module_command" $PROJECTS_FILE)
+
+        cg-trace --xml $(trace_dir $project)/trace.xml --module $MODULE_COMMAND
+    done
+}
+
+db() {
+    for project in $@; do
+        check_project_exists $project
+
+        echo "Creating CodeQL database for '$project"
+
+        DB=$(db_path $project)
+        SRC=$(repo_dir $project)
+        PYTHON_EXTRACTOR=$(codeql resolve extractor --language=python)
+
+        # Source venv so we can extract dependencies
+        source $(venv_dir $project)/bin/activate
+
+        rm -rf "$DB"
+
+        codeql database init --source-root="$SRC" --language=python "$DB"
+        codeql database trace-command --working-dir="$SRC" "$DB" "$PYTHON_EXTRACTOR/tools/autobuild.sh"
+        codeql database index-files --language xml --include-extension .xml --working-dir="$(trace_dir $project)" "$DB"
+        codeql database finalize "$DB"
+
+        echo "Created database in '$DB'"
+
+        # deactivate venv again
+        deactivate
+    done
+}
+
+all() {
+    for project in $@; do
+        check_project_exists $project
+
+        repo $project
+        setup $project
+        trace $project
+        db $project
+    done
+}
+
+
+COMMAND=${1:-"help"}
+shift
+
+$COMMAND $@

python/tools/recorded-call-graph-metrics/projects.json

@@ -0,0 +1,18 @@
+{
+    "wcwidth": {
+        "repo": "https://github.com/jquast/wcwidth.git",
+        "sha": "b29897e5a1b403a0e36f7fc991614981cbc42475",
+        "module_command": "pytest -c /dev/null",
+        "setup": [
+            "pip install pytest"
+        ]
+    },
+    "youtube-dl": {
+        "repo": "https://github.com/ytdl-org/youtube-dl.git",
+        "sha": "a115e07594ccb7749ca108c889978510c7df126e",
+        "module_command": "nose -v test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py --exclude test_socks.py",
+        "setup": [
+            "pip install nose"
+        ]
+    }
+}