diff --git a/Cargo.lock b/Cargo.lock index 66a9d16..4f1ba2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -257,9 +257,9 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8467f424ab6d70e5daf384289fa68247787effc13901dd0ca46bc9b4a62f1474" dependencies = [ - "gix", "hex", "home", + "http 1.1.0", "memchr", "rustc-hash", "semver", @@ -483,21 +483,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "932dcfbd51320af5f27f1ba02d2e567dec332cac7d2c221ba45d8e767264c4dc" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1600,22 +1585,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper 1.3.1", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.4" @@ -1839,24 +1808,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1907,50 +1858,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "openssl" -version = "0.10.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" -dependencies = [ - "bitflags 2.5.0", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-sys" -version = "0.9.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "overload" version = "0.1.1" @@ -2069,12 +1982,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkg-config" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" - [[package]] name = "platforms" version = "3.4.0" @@ -2281,13 +2188,11 @@ dependencies = [ "http-body-util", "hyper 1.3.1", "hyper-rustls", - "hyper-tls", "hyper-util", "ipnet", "js-sys", "log", "mime", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -2301,7 +2206,6 @@ dependencies = [ "sync_wrapper", "system-configuration", "tokio", - "tokio-native-tls", "tokio-rustls", "tokio-util", "tower-service", @@ -2612,8 +2516,9 @@ dependencies = [ "error_reporter", "font-awesome-as-a-crate", "futures-util", - "gix", "grass", + "http 1.1.0", + "http-body-util", "hyper 0.14.28", "indexmap", "lru_time_cache", @@ -2629,6 +2534,7 @@ dependencies = [ "serde", "serde_urlencoded", "sha-1", + "static_assertions", "tokio", "toml 0.8.13", "tracing", @@ -2885,16 +2791,6 @@ dependencies = [ "syn 2.0.66", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.25.0" @@ -3178,12 +3074,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index 607ec65..c077da1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,13 +16,15 @@ badge = { path = "./libs/badge" } anyhow = "1" cadence = "1" -crates-index = { version = "2", default-features = false, features = ["git"] } +crates-index = "2" derive_more = "0.99" dotenvy = "0.15" +error_reporter = "1" font-awesome-as-a-crate = "0.3" futures-util = { version = "0.3", default-features = false, features = ["std"] } +http = "1" +http-body-util = "0.1" hyper = { version = "0.14.10", features = ["full"] } -error_reporter = "1" indexmap = { version = "2", features = ["serde"] } lru_time_cache = "0.11" maud = "0.26" @@ -30,7 +32,7 @@ once_cell = "1" parking_lot = "0.12" pulldown-cmark = "0.11" relative-path = { version = "1", features = ["serde"] } -reqwest = { version = "0.12", features = ["json"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "http2", "rustls-tls", "gzip"] } route-recognizer = "0.3" rustsec = "0.29" semver = { version = "1.0", features = ["serde"] } @@ -41,9 +43,9 @@ toml = "0.8" tracing = "0.1.30" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -[target.'cfg(any())'.dependencies] -gix = { version = "0.63", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls"] } - [build-dependencies] grass = { version = "0.13", default-features = false } sha-1 = "0.10" + +[dev-dependencies] +static_assertions = "1" diff --git a/src/interactors/crates.rs b/src/interactors/crates.rs index 1a07f5c..1d81fe2 100644 --- a/src/interactors/crates.rs +++ b/src/interactors/crates.rs @@ -9,7 +9,6 @@ use futures_util::FutureExt as _; use hyper::service::Service; use semver::{Version, VersionReq}; use serde::Deserialize; -use tokio::task::spawn_blocking; use crate::{ models::crates::{CrateDep, CrateDeps, CrateName, CratePath, CrateRelease}, @@ -68,10 +67,10 @@ impl QueryCrate { index: ManagedIndex, crate_name: CrateName, ) -> anyhow::Result { - let crate_name2 = crate_name.clone(); - let krate = spawn_blocking(move || index.crate_(crate_name2)) + let krate = index + .crate_(&crate_name) .await? - .ok_or_else(|| anyhow!("crate '{}' not found", crate_name.as_ref()))?; + .ok_or_else(|| anyhow!("crate '{}' not found", crate_name.as_str()))?; convert_pkgs(krate) } diff --git a/src/main.rs b/src/main.rs index c5414ac..799c27d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -79,15 +79,13 @@ async fn main() { let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port); - let index = ManagedIndex::new(); + let index = ManagedIndex::new(client.clone()); - { - let index = index.clone(); - - tokio::spawn(async move { - index.refresh_at_interval(Duration::from_secs(20)).await; - }); - } + // crates index health check + index + .crate_(&"libc".parse().unwrap()) + .await + .expect("crates index startup check should succeed"); let mut engine = Engine::new(client.clone(), index); engine.set_metrics(metrics); diff --git a/src/models/crates.rs b/src/models/crates.rs index 894ae40..0593cd0 100644 --- a/src/models/crates.rs +++ b/src/models/crates.rs @@ -24,6 +24,12 @@ impl CratePath { #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct CrateName(String); +impl CrateName { + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + impl From for String { fn from(crate_name: CrateName) -> String { crate_name.0 diff --git a/src/utils/index.rs b/src/utils/index.rs index 4e58ae3..6838772 100644 --- a/src/utils/index.rs +++ b/src/utils/index.rs @@ -1,54 +1,70 @@ -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; -use anyhow::Result; -use crates_index::{Crate, GitIndex}; -use parking_lot::Mutex; -use tokio::{ - task::spawn_blocking, - time::{self, MissedTickBehavior}, -}; +use crates_index::{Crate, SparseIndex}; use crate::models::crates::CrateName; #[derive(Clone)] pub struct ManagedIndex { - index: Arc>, + index: Arc, + client: reqwest::Client, } impl ManagedIndex { - pub fn new() -> Self { + pub fn new(client: reqwest::Client) -> Self { // the index path is configurable through the `CARGO_HOME` env variable - let index = Arc::new(Mutex::new(GitIndex::new_cargo_default().unwrap())); + let index = Arc::new(SparseIndex::new_cargo_default().unwrap()); - Self { index } + Self { index, client } } - pub fn crate_(&self, crate_name: CrateName) -> Option { - self.index.lock().crate_(crate_name.as_ref()) - } + /// Finds crate by name, returning a list of its versions. + /// + /// # Errors + /// + /// Returns error if HTTP request to crates index fails. + pub async fn crate_(&self, crate_name: &CrateName) -> anyhow::Result> { + let req = self.index.make_cache_request(crate_name.as_str())?; + let req = http_to_reqwest_req(req); - pub async fn refresh_at_interval(&self, update_interval: Duration) { - let mut update_interval = time::interval(update_interval); - update_interval.set_missed_tick_behavior(MissedTickBehavior::Delay); + let res = self.client.execute(req).await?; + let res = reqwest_to_http_res(res).await?; - loop { - if let Err(err) = self.refresh().await { - tracing::error!( - "failed refreshing the crates.io-index, the operation will be retried: {}", - error_reporter::Report::new(err), - ); - } - update_interval.tick().await; - } - } - - async fn refresh(&self) -> Result<(), crates_index::Error> { - let index = Arc::clone(&self.index); - - spawn_blocking(move || index.lock().update()) - .await - .expect("blocking index update task should never panic")?; - - Ok(()) + Ok(self + .index + .parse_cache_response(crate_name.as_str(), res, true)?) } } + +/// Converts an `http` request builder from `crates-index` to a `reqwest` request. +fn http_to_reqwest_req(req: http::request::Builder) -> reqwest::Request { + let req = req + .body(()) + .expect("request from crates_index crate should be valid"); + + let req = http::Request::from_parts(req.into_parts().0, Vec::new()); + + reqwest::Request::try_from(req).expect("request from crates_index crate should be valid") +} + +/// Converts an `http` request builder from `crates-index` to a `reqwest` request. +/// +/// # Errors +/// +/// Returns error if reading HTTP response fails (e.g.: connection is lost while streaming payload). +async fn reqwest_to_http_res(res: reqwest::Response) -> anyhow::Result>> { + use http_body_util::BodyExt as _; + + let (res, body) = http::Response::from(res).into_parts(); + + let body = body.collect().await?.to_bytes().to_vec(); + + Ok(http::Response::from_parts(res, body)) +} + +#[cfg(test)] +mod tests { + use super::*; + + static_assertions::assert_impl_all!(ManagedIndex: Send, Sync); +}