From 146c0f27d477df87efd9bb8480f5e99944a9e6fe Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Mon, 24 Nov 2025 17:48:55 -0800 Subject: [PATCH] Minimal changes to build with Rust 1.91. Approximately: - Replaced `array_chunks` usage with `Itertools::chunks`, which isn't a _totally_ drop-in replacement, but should be sufficient here. - Removed the other nightly flag completely, since it was unused. - Annotated unused functions with `#[expect(unused)]` so they changes either be removed later, or those annotations can be removed when they are in use. - Removed placeholder docblocks in error enums, since they were causing errors. - Accepted various other Clippy suggestions, none of which were particularly interesting. --- Cargo.lock | 14 ++++++++++++-- Cargo.toml | 3 ++- src/analysis.rs | 50 +++++++++++++++++++++++++++---------------------- src/compile.rs | 29 ++++++++++++---------------- src/crate_fs.rs | 17 ++++++++++------- src/db.rs | 4 ++++ src/error.rs | 8 -------- src/index.rs | 24 +++++++++++++----------- src/lib.rs | 2 -- src/main.rs | 11 +++-------- 10 files changed, 84 insertions(+), 78 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc328f5..60e807f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -790,6 +790,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -929,9 +938,9 @@ dependencies = [ [[package]] name = "llvm-sys" -version = "191.0.0" +version = "170.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "893cddf1adf0354b93411e413553dd4daf5c43195d73f1acfa1e394bdd371456" +checksum = "1a5e5f51e78d53a09a9e7763d700d34859f3cc45a8fb9eaad2e8555bc84ff2a0" dependencies = [ "anyhow", "cc", @@ -1086,6 +1095,7 @@ dependencies = [ "env_logger", "flate2", "futures", + "itertools", "lenient_semver", "llvm-ir-analysis", "log", diff --git a/Cargo.toml b/Cargo.toml index cacdef4..400a746 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ llvm-ir-analysis = { path = "llvm-ir-analysis", features = ["llvm-19"] } rustc-demangle = "0.1" #Utility +itertools = "0.14" walkdir = "2.3" crates-index = "0.19" @@ -54,4 +55,4 @@ tar = "0.4" circular-buffer = "0.1" # Db -neo4rs = { version = "0.6", optional = true } \ No newline at end of file +neo4rs = { version = "0.6", optional = true } diff --git a/src/analysis.rs b/src/analysis.rs index dca15b6..368a62e 100644 --- a/src/analysis.rs +++ b/src/analysis.rs @@ -1,4 +1,5 @@ use crate::{db::Db, Error, Roots}; +use itertools::Itertools; use llvm_ir_analysis::{llvm_ir::Module, ModuleAnalysis}; use rayon::prelude::*; use rustc_demangle::demangle; @@ -34,17 +35,20 @@ pub fn extract_calls>(crate_bc_dir: P) -> Result>(crate_bc_dir: P, db: Arc) -> Re // If this crate/version has an invoke, assume its completed and bail if db.has_any_invoke(crate_name, crate_version).await? { - log::trace!("{}-{} Exists, skipping..", crate_name, crate_version); + log::trace!("{crate_name}-{crate_version} Exists, skipping.."); return Ok(()); } - log::trace!("Importing: {}", crate_name); + log::trace!("Importing: {crate_name}"); for (caller, callee) in &calls { let dst_crate = callee.split_once("::").unwrap_or(("NONE", "")).0; @@ -95,8 +99,8 @@ pub async fn export_all_db>(bc_root: P, db: Arc) -> Result<() .filter(|e| e.path().is_dir()) .collect(); - let iter = dirs.iter().array_chunks::<16>(); - for chunk in iter { + let chunks = dirs.iter().chunks(16); + for chunk in &chunks { let tasks: Vec<_> = chunk .into_iter() .map(|c| export_crate_db(c.path(), db.clone())) @@ -145,6 +149,7 @@ impl CountUnsafeResult { } } +#[expect(unused)] pub(crate) async fn count_unsafe_crate_extract( c: Crate, roots: Roots, @@ -192,13 +197,13 @@ pub(crate) async fn count_unsafe_crate_extract( } Ok(()) } + pub(crate) async fn count_unsafe_crate(c: Crate, roots: Roots, db: Arc) -> Result<(), Error> { - let compressed_root = &roots.compressed_root; let sources_root = &roots.sources_root; for v in c.versions() { let crate_fullname = format!("{}-{}", v.name(), v.version()); - let crate_path = sources_root.join(format!("{}", &crate_fullname)); + let crate_path = sources_root.join(&crate_fullname); // Lets work off the tgz for now, since we cant extract // TODO: this needs to be unified to a file driver @@ -215,7 +220,7 @@ pub(crate) async fn count_unsafe_crate(c: Crate, roots: Roots, db: Arc) -> R let unsafe_result: CountUnsafeResult = serde_json::from_str(raw_json).unwrap(); if unsafe_result.has_unsafe() { log::debug!("{} unsafe", &crate_fullname); - db.set_unsafe(v.name(), v.version(), &unsafe_result).await; + db.set_unsafe(v.name(), v.version(), &unsafe_result).await?; //.unwrap(); } } @@ -227,8 +232,8 @@ pub(crate) async fn count_unsafe_crate(c: Crate, roots: Roots, db: Arc) -> R pub(crate) async fn count_unsafe(roots: &Roots, db: Arc) -> Result<(), Error> { let index = crates_index::Index::new_cargo_default().map_err(crate::index::Error::from)?; - let iter = index.crates().array_chunks::<128>(); - for chunk in iter { + let chunks = index.crates().chunks(128); + for chunk in &chunks { let tasks: Vec<_> = chunk .into_iter() .map(|c| count_unsafe_crate(c, roots.clone(), db.clone())) @@ -249,12 +254,13 @@ fn export_crate_csv>(crate_bc_dir: P) -> Result<(), Error> { let mut file = std::fs::OpenOptions::new() .write(true) .create(true) + .truncate(true) .open(crate_bc_dir.as_ref().join("calls.csv")) .unwrap(); - calls.iter().enumerate().for_each(|(_, (src, dst))| { + for (src, dst) in &calls { writeln!(file, "{crate_fullname},{src},{dst}").unwrap(); - }); + } } Ok(()) diff --git a/src/compile.rs b/src/compile.rs index 0582601..84974ec 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -1,6 +1,6 @@ #![allow(clippy::module_name_repetitions)] -use crate::crate_fs::{CrateCache, CrateEntry, CrateFs}; +use crate::crate_fs::CrateFs; use crates_index::{Crate, Index}; use std::{ path::{Path, PathBuf}, @@ -10,22 +10,17 @@ use walkdir::WalkDir; #[derive(thiserror::Error, Debug)] pub enum Error { - /// #[error("IO Error: {0}")] IoError(#[from] std::io::Error), - /// #[error("Crate compilation failed")] CompileFailed(String), - /// #[error("Clean stage failed")] CleanFailure(std::process::Output), - /// #[error("LLVM IR failure: {0}")] + #[expect(unused)] LLVMError(String), - /// #[error("Indexing Error: {0}")] IndexError(#[from] crates_index::Error), - /// #[error("Indexing Error: {0}")] CrateFsError(#[from] crate::crate_fs::Error), } @@ -48,7 +43,7 @@ pub fn clean(path: &Path) -> Result<(), Error> { .output() .unwrap(); - std::fs::remove_dir_all(path.join("target")); + std::fs::remove_dir_all(path.join("target"))?; if output.status.success() { Ok(()) @@ -99,10 +94,10 @@ fn compile_crate>( .output() .unwrap(); - log::trace!("Compiled: {} with result: {:?}", fullname, output); + log::trace!("Compiled: {fullname} with result: {output:?}"); if output.status.success() { - std::fs::create_dir(&output_dir); + std::fs::create_dir(&output_dir)?; // If the compile succeeded, search for emitted .bc files of bytecode and copy them over // to the Roots::bytecode_root directory. @@ -127,14 +122,14 @@ fn compile_crate>( std::str::from_utf8(&output.stdout).unwrap(), std::str::from_utf8(&output.stderr).unwrap() ))); - }; + } Ok(()) } /// Walks the entire `Roots::sources_root` and attempts to compile all crates in parallel. -pub async fn compile_all + Send + Sync>( - mut fs: CrateFs, +pub fn compile_all + Send + Sync>( + fs: CrateFs, bc_root: P, update_only: bool, ) -> Result<(), Error> { @@ -152,7 +147,7 @@ pub async fn compile_all + Send + Sync>( let v = c.highest_version(); let fullname = format!("{}-{}", c.name(), v.version()); - log::trace!("Opening: {}", fullname); + log::trace!("Opening: {fullname}"); if update_only && bc_root.join(&fullname).exists() { log::info!("{} bytecode exists, skipping..", &fullname); @@ -164,20 +159,20 @@ pub async fn compile_all + Send + Sync>( if let Ok(entry) = lock.open(&fullname) { entry.path().to_path_buf() } else { - log::error!("Opening failed on {}", fullname); + log::error!("Opening failed on {fullname}"); return; } }; if let Err(e) = compile_crate(c.name(), v.version(), &cache, &bc_root) { - log::error!("{:?}", e); + log::error!("{e:?}"); } //} }; index .crates_parallel() - .filter_map(|c| c.ok()) + .filter_map(Result::ok) .for_each(|c| { do_crate(c, fs.clone(), bc_root.as_ref().to_path_buf()); }); diff --git a/src/crate_fs.rs b/src/crate_fs.rs index afa5be9..3347d63 100644 --- a/src/crate_fs.rs +++ b/src/crate_fs.rs @@ -1,9 +1,6 @@ #![allow(clippy::module_name_repetitions)] use circular_buffer::CircularBuffer; -use std::{ - path::{Path, PathBuf}, - sync::Mutex, -}; +use std::path::{Path, PathBuf}; /// Top error type returned during any stage of analysis from compile to data import. #[derive(thiserror::Error, Debug)] @@ -42,10 +39,12 @@ impl CrateEntry { &self.full_name } + #[expect(unused)] pub fn name(&self) -> &str { self.full_name.rsplit_once('-').unwrap().0 } + #[expect(unused)] pub fn version(&self) -> &str { self.full_name.rsplit_once('-').unwrap().1 } @@ -65,6 +64,7 @@ where #[derive(Debug)] pub struct CrateCache { + #[expect(unused)] src_crate_file: PathBuf, extracted_path: PathBuf, no_delete: bool, @@ -113,7 +113,7 @@ impl CrateCache { } impl Drop for CrateCache { fn drop(&mut self) { - log::trace!("dropping {:?}", self); + log::trace!("dropping {self:?}"); if !self.no_delete { std::fs::remove_dir_all(&self.extracted_path).unwrap(); } @@ -146,6 +146,7 @@ impl CrateFsConfig { pub struct CrateFs { cache: Box>, + #[expect(unused)] index: crates_index::Index, config: CrateFsConfig, } @@ -171,6 +172,8 @@ impl CrateFs { }, ) } + + #[expect(unused)] pub fn close>(&mut self, fullname: S) -> Result<(), Error> { let entry = CrateEntry::new(fullname.as_ref().to_string())?; @@ -205,6 +208,7 @@ impl CrateFs { } } + #[expect(unused)] pub fn config(&self) -> &CrateFsConfig { &self.config } @@ -212,8 +216,7 @@ impl CrateFs { #[cfg(test)] mod tests { - use super::*; - + #[expect(unused)] fn init_logging() { // capture log messages with test harness let _ = env_logger::builder().is_test(true).try_init(); diff --git a/src/db.rs b/src/db.rs index 2f5c62d..236eeef 100644 --- a/src/db.rs +++ b/src/db.rs @@ -19,6 +19,7 @@ pub struct Db { } impl Db { #[allow(clippy::must_use_candidate)] + #[expect(unused)] pub fn inner(&self) -> Arc { self.conn.clone() } @@ -214,6 +215,7 @@ impl Db { /// # Errors /// This function will return an `painter::db::Error` in the event of a database error. #[allow(clippy::similar_names)] + #[expect(unused)] pub async fn upsert_invoke( &self, caller: &str, @@ -261,6 +263,7 @@ impl Db { /// a new Node is not returned during insertion. /// # Errors /// This function will return an `painter::db::Error` in the event of a database error. + #[expect(unused)] pub async fn upsert_crate_version<'a, I, S1, S2, S3, S4, S5>( &self, name: &str, @@ -477,6 +480,7 @@ impl Db { /// /// # Errors /// + #[expect(unused)] pub async fn crate_version_exists(&self, name: S1, version: S2) -> Result where S1: AsRef, diff --git a/src/error.rs b/src/error.rs index aa1cb2f..71f02cd 100644 --- a/src/error.rs +++ b/src/error.rs @@ -3,30 +3,22 @@ use crate::{crate_fs, db, index}; /// Top error type returned during any stage of analysis from compile to data import. #[derive(thiserror::Error, Debug)] pub enum Error { - /// #[error("IO Error: {0}")] IoError(#[from] std::io::Error), #[error( "Crate name contained invalid characters or did not match the NAME-VER format. Name: {0}" )] - /// CrateNameError(String), - /// #[error("LLVM IR failure: {0}")] LLVMError(String), - /// #[error("Database Error: {0}")] DbError(#[from] db::Error), - /// #[error("Indexing Error: {0}")] IndexError(#[from] index::Error), - /// #[error("Indexing Error: {0}")] CrateFsError(#[from] crate_fs::Error), - /// #[error("MissingCompressedPath")] MissingCompressedPath, - /// #[error("MissingExtractedSourcesPath")] MissingExtractedSourcesPath, } diff --git a/src/index.rs b/src/index.rs index cb3c8e6..333ecdb 100644 --- a/src/index.rs +++ b/src/index.rs @@ -1,5 +1,6 @@ use crate::db::Db; use crates_index::Crate; +use itertools::Itertools; use std::sync::Arc; #[derive(thiserror::Error, Debug)] @@ -38,8 +39,8 @@ pub async fn update_missing_versions(conn: Arc) -> Result<(), Error> { } }; - let iter = index.crates().array_chunks::<128>(); - for chunk in iter { + let chunks = index.crates().chunks(128); + for chunk in &chunks { let tasks: Vec<_> = chunk .into_iter() .map(|c| do_crate(c, conn.clone())) @@ -56,6 +57,7 @@ pub async fn update_missing_versions(conn: Arc) -> Result<(), Error> { /// asdf /// # Errors /// asdf +#[expect(unused)] pub async fn update_missing_crates(conn: Arc) -> Result<(), Error> { let index = crates_index::Index::new_cargo_default()?; @@ -65,14 +67,14 @@ pub async fn update_missing_crates(conn: Arc) -> Result<(), Error> { println!("Missing crate: {}", c.name()); if let Err(e) = insert_fresh_crate(c.clone(), db.clone()).await { log::error!("Failed crate: {}", c.name()); - log::error!("Failed crate: {}", e); + log::error!("Failed crate: {e}"); } } } }; - let iter = index.crates().array_chunks::<128>(); - for chunk in iter { + let chunks = index.crates().chunks(128); + for chunk in &chunks { let tasks: Vec<_> = chunk .into_iter() .map(|c| do_crate(c, conn.clone())) @@ -122,8 +124,8 @@ pub async fn insert_fresh_crate(c: Crate, db: Arc) -> Result<(), Error> { pub async fn create_fresh_db(conn: Arc) -> Result<(), Error> { let index = crates_index::Index::new_cargo_default()?; - let iter = index.crates().array_chunks::<64>(); - for chunk in iter { + let chunks = index.crates().chunks(64); + for chunk in &chunks { let tasks: Vec<_> = chunk .into_iter() .map(|c| insert_fresh_crate(c, conn.clone())) @@ -145,17 +147,17 @@ pub async fn set_latest_versions(conn: Arc) -> Result<(), Error> { let do_crate = |c: Crate, db: Arc| async move { let latest = c.highest_version(); - db.set_latest(c.name(), latest.version()).await; + db.set_latest(c.name(), latest.version()).await }; - let iter = index.crates().array_chunks::<64>(); - for chunk in iter { + let chunks = index.crates().chunks(64); + for chunk in &chunks { let tasks: Vec<_> = chunk .into_iter() .map(|c| do_crate(c, conn.clone())) .collect(); - futures::future::join_all(tasks).await; + futures::future::try_join_all(tasks).await?; } Ok(()) diff --git a/src/lib.rs b/src/lib.rs index f7a19f5..edf270a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,10 +6,8 @@ use std::path::Path; /// Top error type returned during any stage of analysis from compile to data import. #[derive(thiserror::Error, Debug)] pub enum Error { - /// #[error("IO Error: {0}")] IoError(#[from] std::io::Error), - /// #[error("LLVM IR failure: {0}")] LLVMError(String), } diff --git a/src/main.rs b/src/main.rs index 115c162..de039e9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,5 @@ #![deny(clippy::all, clippy::pedantic)] #![allow(clippy::enum_variant_names)] -#![feature(string_remove_matches)] -#![feature(iter_array_chunks)] mod analysis; mod compile; @@ -38,6 +36,7 @@ struct Args { /// `bytecodes_root`: A location that all bytecodes will be emitted via `rustc`, distributed in folders /// in the format of `sources_root/-` #[derive(clap::Args, Debug, Clone)] +#[allow(clippy::struct_field_names)] struct Roots { /// Root directory containing the extracted sources tree. #[arg(short = 's', value_name = "DIR", value_hint = clap::ValueHint::DirPath)] @@ -145,7 +144,7 @@ async fn main() -> Result<(), Error> { env_logger::init(); let args = Args::parse(); - log::trace!("{:?}", args); + log::trace!("{args:?}"); match args.command { Command::CreateFreshDb { @@ -174,10 +173,7 @@ async fn main() -> Result<(), Error> { //index::update_missing_crates(db.clone()).await?; index::set_latest_versions(db.clone()).await?; } - Command::Compile { - crate_fullname, - roots, - } => { + Command::Compile { .. } => { // let sources = roots.get_crate_sources()?; //compile_crate(&sources[&crate_fullname], roots.bytecodes_root.unwrap())?; } @@ -187,7 +183,6 @@ async fn main() -> Result<(), Error> { roots.bytecodes_root.unwrap(), update_only, ) - .await .unwrap(); } Command::CountUnsafe {