From 07b475ccc3810782241019c736b3da2ae6fa6cac Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Fri, 17 Jan 2025 10:38:58 +0100 Subject: [PATCH 1/4] Rust: extract sources of crates --- rust/tools/BUILD.bazel | 2 +- rust/tools/autobuild.py | 27 +++++++++++++++++++++++++++ rust/tools/autobuild.sh | 4 +++- 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 rust/tools/autobuild.py diff --git a/rust/tools/BUILD.bazel b/rust/tools/BUILD.bazel index 92a97109edee..6b1c42d4a13b 100644 --- a/rust/tools/BUILD.bazel +++ b/rust/tools/BUILD.bazel @@ -2,7 +2,7 @@ load("//misc/bazel:pkg.bzl", "codeql_pkg_files") codeql_pkg_files( name = "tools", - srcs = glob(["*.cmd"]), + srcs = glob(["*.cmd", "autobuild.py"]), exes = glob(["*.sh"]), visibility = ["//rust:__pkg__"], ) diff --git a/rust/tools/autobuild.py b/rust/tools/autobuild.py new file mode 100644 index 000000000000..357f8350aabe --- /dev/null +++ b/rust/tools/autobuild.py @@ -0,0 +1,27 @@ + +import os +import json +import subprocess + +""" +run "cargo metadata --format-version=1" +""" + + +def get_cargo_metadata(): + metadata = json.loads(subprocess.check_output( + ["cargo", "metadata", "--format-version=1"])) + return metadata + + +CODEQL_EXTRACTOR_RUST_ROOT = os.environ.get("CODEQL_EXTRACTOR_RUST_ROOT") +CODEQL_PLATFORM = os.environ.get("CODEQL_PLATFORM") +metadata = get_cargo_metadata() +for package in metadata['packages']: + for target in package['targets']: + if 'lib' in target['kind']: + src_path = target['src_path'] + dir = os.path.dirname(src_path) + autobuild = "{root}/tools/{platform}/autobuild".format( + root=CODEQL_EXTRACTOR_RUST_ROOT, platform=CODEQL_PLATFORM) + subprocess.run([autobuild], cwd=dir) diff --git a/rust/tools/autobuild.sh b/rust/tools/autobuild.sh index 6d78ecf6d1f1..483f44c8ca67 100755 --- a/rust/tools/autobuild.sh +++ b/rust/tools/autobuild.sh @@ -3,4 +3,6 @@ set -eu export RUST_BACKTRACE=1 -exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/autobuild" +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +exec /usr/bin/env python "${SCRIPT_DIR}/autobuild.py" \ No newline at end of file From dc847e4826ebfe181f726b0e97e5e6d988fa93f7 Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Fri, 17 Jan 2025 11:10:22 +0100 Subject: [PATCH 2/4] Rust: dump cargo metadata json data --- rust/tools/autobuild.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/rust/tools/autobuild.py b/rust/tools/autobuild.py index 357f8350aabe..91a3d162067f 100644 --- a/rust/tools/autobuild.py +++ b/rust/tools/autobuild.py @@ -16,7 +16,16 @@ def get_cargo_metadata(): CODEQL_EXTRACTOR_RUST_ROOT = os.environ.get("CODEQL_EXTRACTOR_RUST_ROOT") CODEQL_PLATFORM = os.environ.get("CODEQL_PLATFORM") +database = os.environ.get("CODEQL_EXTRACTOR_RUST_WIP_DATABASE") +scratch_dir = os.environ.get("CODEQL_EXTRACTOR_RUST_SCRATCH_DIR") metadata = get_cargo_metadata() +metadata_file = os.path.join(scratch_dir, "metadata_file.yaml") +with open(metadata_file, "w") as f: + f.write("---\n") + f.write(json.dumps(metadata, indent=4)) + +subprocess.run(["codeql", "database", "index-files", database, + "-lyaml", "--working-dir", scratch_dir, "--include", "metadata_file.yaml"]) for package in metadata['packages']: for target in package['targets']: if 'lib' in target['kind']: From 1964daf71c0f96c0d88f2b3cd9334988dd6f9bea Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Mon, 20 Jan 2025 13:11:27 +0100 Subject: [PATCH 3/4] Rust: exclude all function, const, and static bodies for dependencies --- rust/extractor/src/translate/base.rs | 31 ++++++++++++++++++++++++++-- rust/tools/autobuild.py | 20 +++++++++++++----- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/rust/extractor/src/translate/base.rs b/rust/extractor/src/translate/base.rs index 584d1d0be5cd..f29371ca9b41 100644 --- a/rust/extractor/src/translate/base.rs +++ b/rust/extractor/src/translate/base.rs @@ -16,7 +16,7 @@ use ra_ap_ide_db::RootDatabase; use ra_ap_ide_db::line_index::{LineCol, LineIndex}; use ra_ap_parser::SyntaxKind; use ra_ap_span::TextSize; -use ra_ap_syntax::ast::HasName; +use ra_ap_syntax::ast::{Const, Fn, HasName, Static}; use ra_ap_syntax::{ AstNode, NodeOrToken, SyntaxElementChildren, SyntaxError, SyntaxNode, SyntaxToken, TextRange, ast, @@ -598,7 +598,34 @@ impl<'a> Translator<'a> { })(); } - pub(crate) fn should_be_excluded(&self, item: &impl ast::HasAttrs) -> bool { + pub(crate) fn should_be_excluded(&self, item: &(impl ast::HasAttrs + ast::AstNode)) -> bool { + if "true" + == std::env::var("CODEQL_EXTRACTOR_RUST_OPTION_EXCLUDE_BODIES").unwrap_or_default() + { + let syntax = item.syntax(); + if let Some(body) = syntax.parent().and_then(Fn::cast).and_then(|x| x.body()) { + if body.syntax() == syntax { + tracing::debug!("Skipping Fn body"); + return true; + } + } + if let Some(body) = syntax.parent().and_then(Const::cast).and_then(|x| x.body()) { + if body.syntax() == syntax { + tracing::debug!("Skipping Const body"); + return true; + } + } + if let Some(body) = syntax + .parent() + .and_then(Static::cast) + .and_then(|x| x.body()) + { + if body.syntax() == syntax { + tracing::debug!("Skipping Static body"); + return true; + } + } + } self.semantics.is_some_and(|sema| { item.attrs().any(|attr| { attr.as_simple_call().is_some_and(|(name, tokens)| { diff --git a/rust/tools/autobuild.py b/rust/tools/autobuild.py index 91a3d162067f..12d1a1d658a0 100644 --- a/rust/tools/autobuild.py +++ b/rust/tools/autobuild.py @@ -20,17 +20,27 @@ def get_cargo_metadata(): scratch_dir = os.environ.get("CODEQL_EXTRACTOR_RUST_SCRATCH_DIR") metadata = get_cargo_metadata() metadata_file = os.path.join(scratch_dir, "metadata_file.yaml") +os.makedirs(scratch_dir, exist_ok=True) with open(metadata_file, "w") as f: f.write("---\n") f.write(json.dumps(metadata, indent=4)) subprocess.run(["codeql", "database", "index-files", database, "-lyaml", "--working-dir", scratch_dir, "--include", "metadata_file.yaml"]) +paths = set() for package in metadata['packages']: for target in package['targets']: - if 'lib' in target['kind']: + if 'lib' in target['kind'] or 'bin' in target['kind']: src_path = target['src_path'] - dir = os.path.dirname(src_path) - autobuild = "{root}/tools/{platform}/autobuild".format( - root=CODEQL_EXTRACTOR_RUST_ROOT, platform=CODEQL_PLATFORM) - subprocess.run([autobuild], cwd=dir) + paths.add(os.path.dirname(src_path)) + +autobuild = "{root}/tools/{platform}/autobuild".format( + root=CODEQL_EXTRACTOR_RUST_ROOT, platform=CODEQL_PLATFORM) + +for path in paths: + if not path.startswith(os.getcwd()): + env = os.environ.copy() + env['CODEQL_EXTRACTOR_RUST_OPTION_EXCLUDE_BODIES'] = "true" + else: + env = None + subprocess.run([autobuild], cwd=path, env=env) From f6bd5381d2926c059300818a539d172af28b8a9b Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Fri, 7 Feb 2025 16:02:03 +0100 Subject: [PATCH 4/4] Extract dependencies for QL tests --- rust/tools/autobuild.sh | 2 +- rust/tools/qltest.sh | 35 ++++++++++++++++++++++++----------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/rust/tools/autobuild.sh b/rust/tools/autobuild.sh index 483f44c8ca67..e649294abfb3 100755 --- a/rust/tools/autobuild.sh +++ b/rust/tools/autobuild.sh @@ -5,4 +5,4 @@ set -eu export RUST_BACKTRACE=1 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -exec /usr/bin/env python "${SCRIPT_DIR}/autobuild.py" \ No newline at end of file +exec /usr/bin/env python3 "${SCRIPT_DIR}/autobuild.py" \ No newline at end of file diff --git a/rust/tools/qltest.sh b/rust/tools/qltest.sh index 0eccf5a73087..14e620361dcf 100755 --- a/rust/tools/qltest.sh +++ b/rust/tools/qltest.sh @@ -7,15 +7,28 @@ export RUST_BACKTRACE=full QLTEST_LOG="$CODEQL_EXTRACTOR_RUST_LOG_DIR/qltest.log" QLTEST_COLORED_LOG="$CODEQL_EXTRACTOR_RUST_SCRATCH_DIR/qltest.log" dirname "$QLTEST_LOG" "$QLTEST_COLORED_LOG" | xargs mkdir -p -# put full-color output on the side, but remove the color codes from the log file -# also, print (colored) output only in case of failure -if ! "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" \ - --qltest \ - --logging-verbosity=progress+ \ - 2>&1 \ - | tee "$QLTEST_COLORED_LOG" \ - | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' \ - > "$QLTEST_LOG"; then - cat "$QLTEST_COLORED_LOG" - exit 1 + +if [ -f "Cargo.lock" ]; then + SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + if ! /usr/bin/env python3 "${SCRIPT_DIR}/autobuild.py" \ + 2>&1 \ + | tee "$QLTEST_COLORED_LOG" \ + | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' \ + > "$QLTEST_LOG"; then + cat "$QLTEST_COLORED_LOG" + exit 1 + fi +else + # put full-color output on the side, but remove the color codes from the log file + # also, print (colored) output only in case of failure + if ! "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" \ + --qltest \ + --logging-verbosity=progress+ \ + 2>&1 \ + | tee "$QLTEST_COLORED_LOG" \ + | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' \ + > "$QLTEST_LOG"; then + cat "$QLTEST_COLORED_LOG" + exit 1 + fi fi