refactor: use sparse crates index

Cargo.lock

@@ -257,9 +257,9 @@ version = "2.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8467f424ab6d70e5daf384289fa68247787effc13901dd0ca46bc9b4a62f1474"
 dependencies = [
- "gix",
  "hex",
  "home",
+ "http 1.1.0",
  "memchr",
  "rustc-hash",
  "semver",
@@ -483,21 +483,6 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "932dcfbd51320af5f27f1ba02d2e567dec332cac7d2c221ba45d8e767264c4dc"
 
-[[package]]
-name = "foreign-types"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
-dependencies = [
- "foreign-types-shared",
-]
-
-[[package]]
-name = "foreign-types-shared"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
-
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -1600,22 +1585,6 @@ dependencies = [
  "tower-service",
 ]
 
-[[package]]
-name = "hyper-tls"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
-dependencies = [
- "bytes",
- "http-body-util",
- "hyper 1.3.1",
- "hyper-util",
- "native-tls",
- "tokio",
- "tokio-native-tls",
- "tower-service",
-]
-
 [[package]]
 name = "hyper-util"
 version = "0.1.4"
@@ -1839,24 +1808,6 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
-[[package]]
-name = "native-tls"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
-dependencies = [
- "lazy_static",
- "libc",
- "log",
- "openssl",
- "openssl-probe",
- "openssl-sys",
- "schannel",
- "security-framework",
- "security-framework-sys",
- "tempfile",
-]
-
 [[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
@@ -1907,50 +1858,12 @@ version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
-[[package]]
-name = "openssl"
-version = "0.10.64"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f"
-dependencies = [
- "bitflags 2.5.0",
- "cfg-if",
- "foreign-types",
- "libc",
- "once_cell",
- "openssl-macros",
- "openssl-sys",
-]
-
-[[package]]
-name = "openssl-macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "openssl-probe"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
-[[package]]
-name = "openssl-sys"
-version = "0.9.102"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
- "vcpkg",
-]
-
 [[package]]
 name = "overload"
 version = "0.1.1"
@@ -2069,12 +1982,6 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
-[[package]]
-name = "pkg-config"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
-
 [[package]]
 name = "platforms"
 version = "3.4.0"
@@ -2281,13 +2188,11 @@ dependencies = [
  "http-body-util",
  "hyper 1.3.1",
  "hyper-rustls",
- "hyper-tls",
  "hyper-util",
  "ipnet",
  "js-sys",
  "log",
  "mime",
- "native-tls",
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
@@ -2301,7 +2206,6 @@ dependencies = [
  "sync_wrapper",
  "system-configuration",
  "tokio",
- "tokio-native-tls",
  "tokio-rustls",
  "tokio-util",
  "tower-service",
@@ -2612,8 +2516,9 @@ dependencies = [
  "error_reporter",
  "font-awesome-as-a-crate",
  "futures-util",
- "gix",
  "grass",
+ "http 1.1.0",
+ "http-body-util",
  "hyper 0.14.28",
  "indexmap",
  "lru_time_cache",
@@ -2629,6 +2534,7 @@ dependencies = [
  "serde",
  "serde_urlencoded",
  "sha-1",
+ "static_assertions",
  "tokio",
  "toml 0.8.13",
  "tracing",
@@ -2885,16 +2791,6 @@ dependencies = [
  "syn 2.0.66",
 ]
 
-[[package]]
-name = "tokio-native-tls"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
-dependencies = [
- "native-tls",
- "tokio",
-]
-
 [[package]]
 name = "tokio-rustls"
 version = "0.25.0"
@@ -3178,12 +3074,6 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
-[[package]]
-name = "vcpkg"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
-
 [[package]]
 name = "version_check"
 version = "0.9.4"

Cargo.toml

@@ -16,13 +16,15 @@ badge = { path = "./libs/badge" }
 
 anyhow = "1"
 cadence = "1"
-crates-index = { version = "2", default-features = false, features = ["git"] }
+crates-index = "2"
 derive_more = "0.99"
 dotenvy = "0.15"
+error_reporter = "1"
 font-awesome-as-a-crate = "0.3"
 futures-util = { version = "0.3", default-features = false, features = ["std"] }
+http = "1"
+http-body-util = "0.1"
 hyper = { version = "0.14.10", features = ["full"] }
-error_reporter = "1"
 indexmap = { version = "2", features = ["serde"] }
 lru_time_cache = "0.11"
 maud = "0.26"
@@ -30,7 +32,7 @@ once_cell = "1"
 parking_lot = "0.12"
 pulldown-cmark = "0.11"
 relative-path = { version = "1", features = ["serde"] }
-reqwest = { version = "0.12", features = ["json"] }
+reqwest = { version = "0.12", default-features = false, features = ["json", "http2", "rustls-tls", "gzip"] }
 route-recognizer = "0.3"
 rustsec = "0.29"
 semver = { version = "1.0", features = ["serde"] }
@@ -41,9 +43,9 @@ toml = "0.8"
 tracing = "0.1.30"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
-[target.'cfg(any())'.dependencies]
-gix = { version = "0.63", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls"] }
-
 [build-dependencies]
 grass = { version = "0.13", default-features = false }
 sha-1 = "0.10"
+
+[dev-dependencies]
+static_assertions = "1"

src/interactors/crates.rs

@@ -9,7 +9,6 @@ use futures_util::FutureExt as _;
 use hyper::service::Service;
 use semver::{Version, VersionReq};
 use serde::Deserialize;
-use tokio::task::spawn_blocking;
 
 use crate::{
     models::crates::{CrateDep, CrateDeps, CrateName, CratePath, CrateRelease},
@@ -68,10 +67,10 @@ impl QueryCrate {
         index: ManagedIndex,
         crate_name: CrateName,
     ) -> anyhow::Result<QueryCrateResponse> {
-        let crate_name2 = crate_name.clone();
+        let krate = index
-        let krate = spawn_blocking(move || index.crate_(crate_name2))
+            .crate_(&crate_name)
             .await?
-            .ok_or_else(|| anyhow!("crate '{}' not found", crate_name.as_ref()))?;
+            .ok_or_else(|| anyhow!("crate '{}' not found", crate_name.as_str()))?;
 
         convert_pkgs(krate)
     }

src/main.rs

@@ -79,15 +79,13 @@ async fn main() {
 
     let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port);
 
-    let index = ManagedIndex::new();
+    let index = ManagedIndex::new(client.clone());
 
-    {
+    // crates index health check
-        let index = index.clone();
+    index
-
+        .crate_(&"libc".parse().unwrap())
-        tokio::spawn(async move {
+        .await
-            index.refresh_at_interval(Duration::from_secs(20)).await;
+        .expect("crates index startup check should succeed");
-        });
-    }
 
     let mut engine = Engine::new(client.clone(), index);
     engine.set_metrics(metrics);

src/models/crates.rs

@@ -24,6 +24,12 @@ impl CratePath {
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct CrateName(String);
 
+impl CrateName {
+    pub(crate) fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
 impl From<CrateName> for String {
     fn from(crate_name: CrateName) -> String {
         crate_name.0

src/utils/index.rs

@@ -1,54 +1,70 @@
-use std::{sync::Arc, time::Duration};
+use std::sync::Arc;
 
-use anyhow::Result;
+use crates_index::{Crate, SparseIndex};
-use crates_index::{Crate, GitIndex};
-use parking_lot::Mutex;
-use tokio::{
-    task::spawn_blocking,
-    time::{self, MissedTickBehavior},
-};
 
 use crate::models::crates::CrateName;
 
 #[derive(Clone)]
 pub struct ManagedIndex {
-    index: Arc<Mutex<GitIndex>>,
+    index: Arc<SparseIndex>,
+    client: reqwest::Client,
 }
 
 impl ManagedIndex {
-    pub fn new() -> Self {
+    pub fn new(client: reqwest::Client) -> Self {
         // the index path is configurable through the `CARGO_HOME` env variable
-        let index = Arc::new(Mutex::new(GitIndex::new_cargo_default().unwrap()));
+        let index = Arc::new(SparseIndex::new_cargo_default().unwrap());
 
-        Self { index }
+        Self { index, client }
     }
 
-    pub fn crate_(&self, crate_name: CrateName) -> Option<Crate> {
+    /// Finds crate by name, returning a list of its versions.
-        self.index.lock().crate_(crate_name.as_ref())
+    ///
-    }
+    /// # Errors
+    ///
+    /// Returns error if HTTP request to crates index fails.
+    pub async fn crate_(&self, crate_name: &CrateName) -> anyhow::Result<Option<Crate>> {
+        let req = self.index.make_cache_request(crate_name.as_str())?;
+        let req = http_to_reqwest_req(req);
 
-    pub async fn refresh_at_interval(&self, update_interval: Duration) {
+        let res = self.client.execute(req).await?;
-        let mut update_interval = time::interval(update_interval);
+        let res = reqwest_to_http_res(res).await?;
-        update_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
 
-        loop {
+        Ok(self
-            if let Err(err) = self.refresh().await {
+            .index
-                tracing::error!(
+            .parse_cache_response(crate_name.as_str(), res, true)?)
-                    "failed refreshing the crates.io-index, the operation will be retried: {}",
-                    error_reporter::Report::new(err),
-                );
-            }
-            update_interval.tick().await;
-        }
-    }
-
-    async fn refresh(&self) -> Result<(), crates_index::Error> {
-        let index = Arc::clone(&self.index);
-
-        spawn_blocking(move || index.lock().update())
-            .await
-            .expect("blocking index update task should never panic")?;
-
-        Ok(())
     }
 }
+
+/// Converts an `http` request builder from `crates-index` to a `reqwest` request.
+fn http_to_reqwest_req(req: http::request::Builder) -> reqwest::Request {
+    let req = req
+        .body(())
+        .expect("request from crates_index crate should be valid");
+
+    let req = http::Request::from_parts(req.into_parts().0, Vec::new());
+
+    reqwest::Request::try_from(req).expect("request from crates_index crate should be valid")
+}
+
+/// Converts an `http` request builder from `crates-index` to a `reqwest` request.
+///
+/// # Errors
+///
+/// Returns error if reading HTTP response fails (e.g.: connection is lost while streaming payload).
+async fn reqwest_to_http_res(res: reqwest::Response) -> anyhow::Result<http::Response<Vec<u8>>> {
+    use http_body_util::BodyExt as _;
+
+    let (res, body) = http::Response::from(res).into_parts();
+
+    let body = body.collect().await?.to_bytes().to_vec();
+
+    Ok(http::Response::from_parts(res, body))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    static_assertions::assert_impl_all!(ManagedIndex: Send, Sync);
+}