diff --git a/Cargo.lock b/Cargo.lock index ecf13e9..cba9fe2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -86,9 +86,9 @@ dependencies = [ [[package]] name = "cargo-lock" -version = "7.1.0" +version = "8.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c408da54db4c50d4693f7e649c299bc9de9c23ead86249e5368830bb32a734b" +checksum = "a11310d20f9f27c3823a27dde3463729407898434fd81234b28db08e7559c7ba" dependencies = [ "semver", "serde", @@ -144,15 +144,17 @@ dependencies = [ [[package]] name = "crates-index" -version = "0.17.0" +version = "0.18.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad4af5c8dd9940a497ef4473e6e558b660a4a1b6e5ce2cb9d85454e2aaaf947" +checksum = "0044896374c388ccbf1497dad6384bf6111dbcad9d7069506df7450ce9b62ea3" dependencies = [ "git2", - "glob", "hex", "home", "memchr", + "num_cpus", + "rayon", + "rustc-hash", "semver", "serde", "serde_derive", @@ -170,6 +172,31 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.8" @@ -192,9 +219,9 @@ dependencies = [ [[package]] name = "cvss" -version = "1.0.2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829862dabeab142ae0efd558d42d8fd874659268ccd810809ac6f1ee6bfcbd3f" +checksum = "7ec6a2f799b0e3103192800872de17ee1d39fe0c598628277b9b012f09b4010f" dependencies = [ "serde", ] @@ -243,6 +270,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -376,9 +409,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.13.25" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" +checksum = "d0155506aab710a86160ddb504a480d2964d7ab5b9e62419be69e0032bc5931c" dependencies = [ "bitflags", "libc", @@ -389,12 +422,6 @@ dependencies = [ "url", ] -[[package]] -name = "glob" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" - [[package]] name = "h2" version = "0.3.13" @@ -609,9 +636,9 @@ checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "libgit2-sys" -version = "0.12.26+1.3.0" +version = "0.13.4+1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" +checksum = "d0fa6563431ede25f5cc7f6d803c6afbc1c5d3ad3d4925d12c882bf2b526f5d1" dependencies = [ "cc", "libc", @@ -695,6 +722,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -836,9 +872,9 @@ checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "platforms" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d0eef3571242013a0d5dc84861c3ae4a652e56e12adf8bdc26ff5f8cb34c94" +checksum = "86d1db500905601725f5c3629a5815a2ce7611fe063de279964b451f3edb3532" dependencies = [ "serde", ] @@ -897,6 +933,30 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd249e82c21598a9a426a4e00dd7adc1d640b22445ec8545feef801d1a74c221" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f51245e1e62e1f1629cbfec37b5793bbabcaeb90f30e94d2ba03564687353e4" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.13" @@ -977,6 +1037,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.0" @@ -988,9 +1054,9 @@ dependencies = [ [[package]] name = "rustsec" -version = "0.25.1" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6136976fbabcd3ca37a12ae8ecc3408e8d7a94916d1cabdabd86aa4464e0887" +checksum = "1be23e93b1c670ea8d07dc9eb8a945bd8e760b207ce24c48da9cd68ef20f33eb" dependencies = [ "cargo-lock", "crates-index", @@ -1062,6 +1128,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "security-framework" version = "2.6.1" @@ -1221,12 +1293,14 @@ dependencies = [ [[package]] name = "smartstring" -version = "0.2.10" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e714dff2b33f2321fdcd475b71cec79781a692d846f37f415fb395a1d2bcd48e" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" dependencies = [ + "autocfg", "serde", "static_assertions", + "version_check", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e82c352..5c81ec8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ badge = { path = "./libs/badge" } anyhow = "1" cadence = "0.29" -crates-index = "0.17" +crates-index = "0.18" derive_more = "0.99" font-awesome-as-a-crate = "0.2" futures-util = { version = "0.3", default-features = false, features = ["std"] } @@ -28,7 +28,7 @@ pulldown-cmark = "0.9" relative-path = { version = "1.3", features = ["serde"] } reqwest = { version = "0.11", features = ["json"] } route-recognizer = "0.3" -rustsec = "0.25" +rustsec = "0.26" semver = { version = "1.0", features = ["serde"] } serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -36,7 +36,7 @@ serde_urlencoded = "0.7" slog = "2" slog-async = "2" slog-term = "2" -tokio = { version = "1.14", features = ["rt-multi-thread", "macros"] } +tokio = { version = "1.14", features = ["rt-multi-thread", "macros", "sync", "time"] } toml = "0.5" [build-dependencies] diff --git a/src/engine/machines/analyzer.rs b/src/engine/machines/analyzer.rs index a2e9aae..ca137f1 100644 --- a/src/engine/machines/analyzer.rs +++ b/src/engine/machines/analyzer.rs @@ -40,7 +40,9 @@ impl DependencyAnalyzer { let name: cargo_lock::Name = name.as_ref().parse().unwrap(); let version: cargo_lock::Version = ver.to_string().parse().unwrap(); - let query = database::Query::crate_scope().package_version(name, version); + let query = database::Query::crate_scope() + .package_name(name) + .package_version(version); if let Some(db) = advisory_db { let vulnerabilities: Vec<_> = diff --git a/src/engine/mod.rs b/src/engine/mod.rs index b37a445..05b8790 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -7,7 +7,7 @@ use std::{ use anyhow::{anyhow, Error}; use cadence::{MetricSink, NopMetricSink, StatsdClient}; -use crates_index::Index; + use futures_util::{ future::try_join_all, stream::{self, BoxStream}, @@ -27,6 +27,7 @@ use crate::interactors::RetrieveFileAtPath; use crate::models::crates::{AnalyzedDependencies, CrateName, CratePath, CrateRelease}; use crate::models::repo::{RepoPath, Repository}; use crate::utils::cache::Cache; +use crate::ManagedIndex; mod fut; mod machines; @@ -44,7 +45,7 @@ pub struct Engine { } impl Engine { - pub fn new(client: reqwest::Client, index: Index, logger: Logger) -> Engine { + pub fn new(client: reqwest::Client, index: ManagedIndex, logger: Logger) -> Engine { let metrics = Arc::new(StatsdClient::from_sink("engine", NopMetricSink)); let query_crate = Cache::new( diff --git a/src/interactors/crates.rs b/src/interactors/crates.rs index 56a2a4b..6c04747 100644 --- a/src/interactors/crates.rs +++ b/src/interactors/crates.rs @@ -1,16 +1,17 @@ use std::{fmt, str, task::Context, task::Poll}; use anyhow::{anyhow, Error}; -use crates_index::{Crate, DependencyKind, Index}; +use crates_index::{Crate, DependencyKind}; use futures_util::FutureExt as _; use hyper::service::Service; use semver::{Version, VersionReq}; use serde::Deserialize; + use tokio::task::spawn_blocking; use crate::{ models::crates::{CrateDep, CrateDeps, CrateName, CratePath, CrateRelease}, - BoxFuture, + BoxFuture, ManagedIndex, }; const CRATES_API_BASE_URI: &str = "https://crates.io/api/v1"; @@ -53,17 +54,20 @@ pub struct QueryCrateResponse { #[derive(Clone)] pub struct QueryCrate { - index: Index, + index: ManagedIndex, } impl QueryCrate { - pub fn new(index: Index) -> Self { + pub fn new(index: ManagedIndex) -> Self { Self { index } } - pub async fn query(index: Index, crate_name: CrateName) -> anyhow::Result { + pub async fn query( + index: ManagedIndex, + crate_name: CrateName, + ) -> anyhow::Result { let crate_name2 = crate_name.clone(); - let krate = spawn_blocking(move || index.crate_(crate_name2.as_ref())) + let krate = spawn_blocking(move || index.crate_(crate_name2)) .await? .ok_or_else(|| anyhow!("crate '{}' not found", crate_name.as_ref()))?; diff --git a/src/main.rs b/src/main.rs index 3ac048d..c568f0b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -71,18 +71,15 @@ async fn main() { let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port); - let mut managed_index = ManagedIndex::new(Duration::from_secs(20), logger.clone()); - if let Err(err) = managed_index.initial_clone().await { - error!( - logger, - "failed running initial clone of the crates.io-index: {err}", - ); - } + let index = ManagedIndex::new(logger.clone()); - let index = managed_index.index(); - tokio::spawn(async move { - managed_index.refresh_at_interval().await; - }); + { + let index = index.clone(); + + tokio::spawn(async move { + index.refresh_at_interval(Duration::from_secs(20)).await; + }); + } let mut engine = Engine::new(client.clone(), index, logger.new(o!())); engine.set_metrics(metrics); diff --git a/src/utils/index.rs b/src/utils/index.rs index b602b41..a698821 100644 --- a/src/utils/index.rs +++ b/src/utils/index.rs @@ -1,49 +1,37 @@ +use std::sync::Arc; +use std::sync::Mutex; use std::time::Duration; -use anyhow::{Error, Result}; +use crate::models::crates::CrateName; +use anyhow::Result; +use crates_index::Crate; use crates_index::Index; -use slog::{error, info, Logger}; +use slog::{error, Logger}; use tokio::task::spawn_blocking; -use tokio::time::{self, Interval}; +use tokio::time; +#[derive(Clone)] pub struct ManagedIndex { - index: Index, - update_interval: Interval, + index: Arc>, logger: Logger, } impl ManagedIndex { - pub fn new(update_interval: Duration, logger: Logger) -> Self { + pub fn new(logger: Logger) -> Self { // the index path is configurable through the `CARGO_HOME` env variable - let index = Index::new_cargo_default(); - let update_interval = time::interval(update_interval); - Self { - index, - update_interval, - logger, - } + let index = Arc::new(Mutex::new(Index::new_cargo_default().unwrap())); + Self { index, logger } } - pub fn index(&self) -> Index { - self.index.clone() + pub fn crate_(&self, crate_name: CrateName) -> Option { + let index = self.index.lock().unwrap(); + + index.crate_(crate_name.as_ref()) } - pub async fn initial_clone(&mut self) -> Result<()> { - let index = self.index(); - let logger = self.logger.clone(); + pub async fn refresh_at_interval(&self, update_interval: Duration) { + let mut update_interval = time::interval(update_interval); - spawn_blocking(move || { - if !index.exists() { - info!(logger, "Cloning crates.io-index"); - index.retrieve()?; - } - Ok::<_, Error>(()) - }) - .await??; - Ok(()) - } - - pub async fn refresh_at_interval(&mut self) { loop { if let Err(e) = self.refresh().await { error!( @@ -51,14 +39,19 @@ impl ManagedIndex { "failed refreshing the crates.io-index, the operation will be retried: {}", e ); } - self.update_interval.tick().await; + update_interval.tick().await; } } async fn refresh(&self) -> Result<()> { - let index = self.index(); + let index = Arc::clone(&self.index); - spawn_blocking(move || index.retrieve_or_update()).await??; + spawn_blocking(move || { + let mut index = index.lock().unwrap(); + + index.update() + }) + .await??; Ok(()) } }