Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ llvm-ir-analysis = { path = "llvm-ir-analysis", features = ["llvm-19"] }
rustc-demangle = "0.1"

#Utility
itertools = "0.14"
walkdir = "2.3"
crates-index = "0.19"

Expand All @@ -54,4 +55,4 @@ tar = "0.4"
circular-buffer = "0.1"

# Db
neo4rs = { version = "0.6", optional = true }
neo4rs = { version = "0.6", optional = true }
50 changes: 28 additions & 22 deletions src/analysis.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use crate::{db::Db, Error, Roots};
use itertools::Itertools;
use llvm_ir_analysis::{llvm_ir::Module, ModuleAnalysis};
use rayon::prelude::*;
use rustc_demangle::demangle;
Expand Down Expand Up @@ -34,17 +35,20 @@ pub fn extract_calls<P: AsRef<Path>>(crate_bc_dir: P) -> Result<Vec<(String, Str
let analysis = ModuleAnalysis::new(&module);

let graph = analysis.call_graph();
graph.inner().all_edges().for_each(|(src_raw, dst_raw, _)| {
let src = format!("{:#}", demangle(src_raw));
let dst = format!("{:#}", demangle(dst_raw));

if !BLOCKED_STRINGS
.iter()
.any(|s| src.contains(*s) || dst.contains(*s))
{
calls.push((src, dst));
}
});
graph
.inner()
.all_edges()
.for_each(|(src_raw, dst_raw, ())| {
let src = format!("{:#}", demangle(src_raw));
let dst = format!("{:#}", demangle(dst_raw));

if !BLOCKED_STRINGS
.iter()
.any(|s| src.contains(*s) || dst.contains(*s))
{
calls.push((src, dst));
}
});
}

Ok(calls)
Expand All @@ -67,11 +71,11 @@ pub async fn export_crate_db<P: AsRef<Path>>(crate_bc_dir: P, db: Arc<Db>) -> Re

// If this crate/version has an invoke, assume its completed and bail
if db.has_any_invoke(crate_name, crate_version).await? {
log::trace!("{}-{} Exists, skipping..", crate_name, crate_version);
log::trace!("{crate_name}-{crate_version} Exists, skipping..");
return Ok(());
}

log::trace!("Importing: {}", crate_name);
log::trace!("Importing: {crate_name}");

for (caller, callee) in &calls {
let dst_crate = callee.split_once("::").unwrap_or(("NONE", "")).0;
Expand All @@ -95,8 +99,8 @@ pub async fn export_all_db<P: AsRef<Path>>(bc_root: P, db: Arc<Db>) -> Result<()
.filter(|e| e.path().is_dir())
.collect();

let iter = dirs.iter().array_chunks::<16>();
for chunk in iter {
let chunks = dirs.iter().chunks(16);
for chunk in &chunks {
let tasks: Vec<_> = chunk
.into_iter()
.map(|c| export_crate_db(c.path(), db.clone()))
Expand Down Expand Up @@ -145,6 +149,7 @@ impl CountUnsafeResult {
}
}

#[expect(unused)]
pub(crate) async fn count_unsafe_crate_extract(
c: Crate,
roots: Roots,
Expand Down Expand Up @@ -192,13 +197,13 @@ pub(crate) async fn count_unsafe_crate_extract(
}
Ok(())
}

pub(crate) async fn count_unsafe_crate(c: Crate, roots: Roots, db: Arc<Db>) -> Result<(), Error> {
let compressed_root = &roots.compressed_root;
let sources_root = &roots.sources_root;

for v in c.versions() {
let crate_fullname = format!("{}-{}", v.name(), v.version());
let crate_path = sources_root.join(format!("{}", &crate_fullname));
let crate_path = sources_root.join(&crate_fullname);

// Lets work off the tgz for now, since we cant extract
// TODO: this needs to be unified to a file driver
Expand All @@ -215,7 +220,7 @@ pub(crate) async fn count_unsafe_crate(c: Crate, roots: Roots, db: Arc<Db>) -> R
let unsafe_result: CountUnsafeResult = serde_json::from_str(raw_json).unwrap();
if unsafe_result.has_unsafe() {
log::debug!("{} unsafe", &crate_fullname);
db.set_unsafe(v.name(), v.version(), &unsafe_result).await;
db.set_unsafe(v.name(), v.version(), &unsafe_result).await?;
//.unwrap();
}
}
Expand All @@ -227,8 +232,8 @@ pub(crate) async fn count_unsafe_crate(c: Crate, roots: Roots, db: Arc<Db>) -> R
pub(crate) async fn count_unsafe(roots: &Roots, db: Arc<Db>) -> Result<(), Error> {
let index = crates_index::Index::new_cargo_default().map_err(crate::index::Error::from)?;

let iter = index.crates().array_chunks::<128>();
for chunk in iter {
let chunks = index.crates().chunks(128);
for chunk in &chunks {
let tasks: Vec<_> = chunk
.into_iter()
.map(|c| count_unsafe_crate(c, roots.clone(), db.clone()))
Expand All @@ -249,12 +254,13 @@ fn export_crate_csv<P: AsRef<Path>>(crate_bc_dir: P) -> Result<(), Error> {
let mut file = std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(crate_bc_dir.as_ref().join("calls.csv"))
.unwrap();

calls.iter().enumerate().for_each(|(_, (src, dst))| {
for (src, dst) in &calls {
writeln!(file, "{crate_fullname},{src},{dst}").unwrap();
});
}
}

Ok(())
Expand Down
29 changes: 12 additions & 17 deletions src/compile.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#![allow(clippy::module_name_repetitions)]

use crate::crate_fs::{CrateCache, CrateEntry, CrateFs};
use crate::crate_fs::CrateFs;
use crates_index::{Crate, Index};
use std::{
path::{Path, PathBuf},
Expand All @@ -10,22 +10,17 @@ use walkdir::WalkDir;

#[derive(thiserror::Error, Debug)]
pub enum Error {
///
#[error("IO Error: {0}")]
IoError(#[from] std::io::Error),
///
#[error("Crate compilation failed")]
CompileFailed(String),
///
#[error("Clean stage failed")]
CleanFailure(std::process::Output),
///
#[error("LLVM IR failure: {0}")]
#[expect(unused)]
LLVMError(String),
///
#[error("Indexing Error: {0}")]
IndexError(#[from] crates_index::Error),
///
#[error("Indexing Error: {0}")]
CrateFsError(#[from] crate::crate_fs::Error),
}
Expand All @@ -48,7 +43,7 @@ pub fn clean(path: &Path) -> Result<(), Error> {
.output()
.unwrap();

std::fs::remove_dir_all(path.join("target"));
std::fs::remove_dir_all(path.join("target"))?;

if output.status.success() {
Ok(())
Expand Down Expand Up @@ -99,10 +94,10 @@ fn compile_crate<P: AsRef<Path>>(
.output()
.unwrap();

log::trace!("Compiled: {} with result: {:?}", fullname, output);
log::trace!("Compiled: {fullname} with result: {output:?}");

if output.status.success() {
std::fs::create_dir(&output_dir);
std::fs::create_dir(&output_dir)?;

// If the compile succeeded, search for emitted .bc files of bytecode and copy them over
// to the Roots::bytecode_root directory.
Expand All @@ -127,14 +122,14 @@ fn compile_crate<P: AsRef<Path>>(
std::str::from_utf8(&output.stdout).unwrap(),
std::str::from_utf8(&output.stderr).unwrap()
)));
};
}

Ok(())
}

/// Walks the entire `Roots::sources_root` and attempts to compile all crates in parallel.
pub async fn compile_all<P: AsRef<Path> + Send + Sync>(
mut fs: CrateFs,
pub fn compile_all<P: AsRef<Path> + Send + Sync>(
fs: CrateFs,
bc_root: P,
update_only: bool,
) -> Result<(), Error> {
Expand All @@ -152,7 +147,7 @@ pub async fn compile_all<P: AsRef<Path> + Send + Sync>(
let v = c.highest_version();

let fullname = format!("{}-{}", c.name(), v.version());
log::trace!("Opening: {}", fullname);
log::trace!("Opening: {fullname}");

if update_only && bc_root.join(&fullname).exists() {
log::info!("{} bytecode exists, skipping..", &fullname);
Expand All @@ -164,20 +159,20 @@ pub async fn compile_all<P: AsRef<Path> + Send + Sync>(
if let Ok(entry) = lock.open(&fullname) {
entry.path().to_path_buf()
} else {
log::error!("Opening failed on {}", fullname);
log::error!("Opening failed on {fullname}");
return;
}
};

if let Err(e) = compile_crate(c.name(), v.version(), &cache, &bc_root) {
log::error!("{:?}", e);
log::error!("{e:?}");
}
//}
};

index
.crates_parallel()
.filter_map(|c| c.ok())
.filter_map(Result::ok)
.for_each(|c| {
do_crate(c, fs.clone(), bc_root.as_ref().to_path_buf());
});
Expand Down
17 changes: 10 additions & 7 deletions src/crate_fs.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
#![allow(clippy::module_name_repetitions)]
use circular_buffer::CircularBuffer;
use std::{
path::{Path, PathBuf},
sync::Mutex,
};
use std::path::{Path, PathBuf};

/// Top error type returned during any stage of analysis from compile to data import.
#[derive(thiserror::Error, Debug)]
Expand Down Expand Up @@ -42,10 +39,12 @@ impl CrateEntry {
&self.full_name
}

#[expect(unused)]
pub fn name(&self) -> &str {
self.full_name.rsplit_once('-').unwrap().0
}

#[expect(unused)]
pub fn version(&self) -> &str {
self.full_name.rsplit_once('-').unwrap().1
}
Expand All @@ -65,6 +64,7 @@ where

#[derive(Debug)]
pub struct CrateCache {
#[expect(unused)]
src_crate_file: PathBuf,
extracted_path: PathBuf,
no_delete: bool,
Expand Down Expand Up @@ -113,7 +113,7 @@ impl CrateCache {
}
impl Drop for CrateCache {
fn drop(&mut self) {
log::trace!("dropping {:?}", self);
log::trace!("dropping {self:?}");
if !self.no_delete {
std::fs::remove_dir_all(&self.extracted_path).unwrap();
}
Expand Down Expand Up @@ -146,6 +146,7 @@ impl CrateFsConfig {

pub struct CrateFs {
cache: Box<CircularBuffer<1024, (CrateEntry, CrateCache)>>,
#[expect(unused)]
index: crates_index::Index,
config: CrateFsConfig,
}
Expand All @@ -171,6 +172,8 @@ impl CrateFs {
},
)
}

#[expect(unused)]
pub fn close<S: AsRef<str>>(&mut self, fullname: S) -> Result<(), Error> {
let entry = CrateEntry::new(fullname.as_ref().to_string())?;

Expand Down Expand Up @@ -205,15 +208,15 @@ impl CrateFs {
}
}

#[expect(unused)]
pub fn config(&self) -> &CrateFsConfig {
&self.config
}
}

#[cfg(test)]
mod tests {
use super::*;

#[expect(unused)]
fn init_logging() {
// capture log messages with test harness
let _ = env_logger::builder().is_test(true).try_init();
Expand Down
4 changes: 4 additions & 0 deletions src/db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ pub struct Db {
}
impl Db {
#[allow(clippy::must_use_candidate)]
#[expect(unused)]
pub fn inner(&self) -> Arc<Graph> {
self.conn.clone()
}
Expand Down Expand Up @@ -214,6 +215,7 @@ impl Db {
/// # Errors
/// This function will return an `painter::db::Error` in the event of a database error.
#[allow(clippy::similar_names)]
#[expect(unused)]
pub async fn upsert_invoke(
&self,
caller: &str,
Expand Down Expand Up @@ -261,6 +263,7 @@ impl Db {
/// a new Node is not returned during insertion.
/// # Errors
/// This function will return an `painter::db::Error` in the event of a database error.
#[expect(unused)]
pub async fn upsert_crate_version<'a, I, S1, S2, S3, S4, S5>(
&self,
name: &str,
Expand Down Expand Up @@ -477,6 +480,7 @@ impl Db {
///
/// # Errors
///
#[expect(unused)]
pub async fn crate_version_exists<S1, S2>(&self, name: S1, version: S2) -> Result<bool, Error>
where
S1: AsRef<str>,
Expand Down
Loading