feliix42/deps.rs
1
0
Fork 0
mirror of https://github.com/deps-rs/deps.rs.git synced 2024-11-21 18:06:30 +00:00
Code Issues Projects Releases Packages Wiki Activity

refactor: use sparse crates index

Browse source
This commit is contained in:
Rob Ede 2024-05-27 00:29:46 +01:00
parent 85a077e80d
commit 9a1f07d355
No known key found for this signature in database
GPG key ID: 97C636207D3EF933
6 changed files with 80 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

118
Cargo.lock generated
View file

@ -257,9 +257,9 @@ version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8467f424ab6d70e5daf384289fa68247787effc13901dd0ca46bc9b4a62f1474"
dependencies = [
"gix",
"hex",
"home",
"http 1.1.0",
"memchr",
"rustc-hash",
"semver",
@ -483,21 +483,6 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "932dcfbd51320af5f27f1ba02d2e567dec332cac7d2c221ba45d8e767264c4dc"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "form_urlencoded"
version = "1.2.1"
@ -1600,22 +1585,6 @@ dependencies = [
"tower-service",
]
[[package]]
name = "hyper-tls"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
dependencies = [
"bytes",
"http-body-util",
"hyper 1.3.1",
"hyper-util",
"native-tls",
"tokio",
"tokio-native-tls",
"tower-service",
]
[[package]]
name = "hyper-util"
version = "0.1.4"
@ -1839,24 +1808,6 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "native-tls"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
dependencies = [
"lazy_static",
"libc",
"log",
"openssl",
"openssl-probe",
"openssl-sys",
"schannel",
"security-framework",
"security-framework-sys",
"tempfile",
]
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
@ -1907,50 +1858,12 @@ version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "openssl"
version = "0.10.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f"
dependencies = [
"bitflags 2.5.0",
"cfg-if",
"foreign-types",
"libc",
"once_cell",
"openssl-macros",
"openssl-sys",
]
[[package]]
name = "openssl-macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
]
[[package]]
name = "openssl-probe"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "openssl-sys"
version = "0.9.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "overload"
version = "0.1.1"
@ -2069,12 +1982,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkg-config"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
[[package]]
name = "platforms"
version = "3.4.0"
@ -2281,13 +2188,11 @@ dependencies = [
"http-body-util",
"hyper 1.3.1",
"hyper-rustls",
"hyper-tls",
"hyper-util",
"ipnet",
"js-sys",
"log",
"mime",
"native-tls",
"once_cell",
"percent-encoding",
"pin-project-lite",
@ -2301,7 +2206,6 @@ dependencies = [
"sync_wrapper",
"system-configuration",
"tokio",
"tokio-native-tls",
"tokio-rustls",
"tokio-util",
"tower-service",
@ -2612,8 +2516,9 @@ dependencies = [
"error_reporter",
"font-awesome-as-a-crate",
"futures-util",
"gix",
"grass",
"http 1.1.0",
"http-body-util",
"hyper 0.14.28",
"indexmap",
"lru_time_cache",
@ -2629,6 +2534,7 @@ dependencies = [
"serde",
"serde_urlencoded",
"sha-1",
"static_assertions",
"tokio",
"toml 0.8.13",
"tracing",
@ -2885,16 +2791,6 @@ dependencies = [
"syn 2.0.66",
]
[[package]]
name = "tokio-native-tls"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
dependencies = [
"native-tls",
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.25.0"
@ -3178,12 +3074,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
version = "0.9.4"

14
Cargo.toml
View file

@ -16,13 +16,15 @@ badge = { path = "./libs/badge" }
anyhow = "1"
cadence = "1"
crates-index = { version = "2", default-features = false, features = ["git"] }
crates-index = "2"
derive_more = "0.99"
dotenvy = "0.15"
error_reporter = "1"
font-awesome-as-a-crate = "0.3"
futures-util = { version = "0.3", default-features = false, features = ["std"] }
http = "1"
http-body-util = "0.1"
hyper = { version = "0.14.10", features = ["full"] }
error_reporter = "1"
indexmap = { version = "2", features = ["serde"] }
lru_time_cache = "0.11"
maud = "0.26"
@ -30,7 +32,7 @@ once_cell = "1"
parking_lot = "0.12"
pulldown-cmark = "0.11"
relative-path = { version = "1", features = ["serde"] }
reqwest = { version = "0.12", features = ["json"] }
reqwest = { version = "0.12", default-features = false, features = ["json", "http2", "rustls-tls", "gzip"] }
route-recognizer = "0.3"
rustsec = "0.29"
semver = { version = "1.0", features = ["serde"] }
@ -41,9 +43,9 @@ toml = "0.8"
tracing = "0.1.30"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
[target.'cfg(any())'.dependencies]
gix = { version = "0.63", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls"] }
[build-dependencies]
grass = { version = "0.13", default-features = false }
sha-1 = "0.10"
[dev-dependencies]
static_assertions = "1"

7
src/interactors/crates.rs
View file

@ -9,7 +9,6 @@ use futures_util::FutureExt as _;
use hyper::service::Service;
use semver::{Version, VersionReq};
use serde::Deserialize;
use tokio::task::spawn_blocking;
use crate::{
models::crates::{CrateDep, CrateDeps, CrateName, CratePath, CrateRelease},
@ -68,10 +67,10 @@ impl QueryCrate {
index: ManagedIndex,
crate_name: CrateName,
) -> anyhow::Result<QueryCrateResponse> {
let crate_name2 = crate_name.clone();
let krate = spawn_blocking(move || index.crate_(crate_name2))
let krate = index
.crate_(&crate_name)
.await?
.ok_or_else(|| anyhow!("crate '{}' not found", crate_name.as_ref()))?;
.ok_or_else(|| anyhow!("crate '{}' not found", crate_name.as_str()))?;
convert_pkgs(krate)
}

14
src/main.rs
View file

@ -79,15 +79,13 @@ async fn main() {
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port);
let index = ManagedIndex::new();
let index = ManagedIndex::new(client.clone());
{
let index = index.clone();
tokio::spawn(async move {
index.refresh_at_interval(Duration::from_secs(20)).await;
});
}
// crates index health check
index
.crate_(&"libc".parse().unwrap())
.await
.expect("crates index startup check should succeed");
let mut engine = Engine::new(client.clone(), index);
engine.set_metrics(metrics);

6
src/models/crates.rs
View file

@ -24,6 +24,12 @@ impl CratePath {
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct CrateName(String);
impl CrateName {
pub(crate) fn as_str(&self) -> &str {
&self.0
}
}
impl From<CrateName> for String {
fn from(crate_name: CrateName) -> String {
crate_name.0

80
src/utils/index.rs
View file

@ -1,54 +1,70 @@
use std::{sync::Arc, time::Duration};
use std::sync::Arc;
use anyhow::Result;
use crates_index::{Crate, GitIndex};
use parking_lot::Mutex;
use tokio::{
task::spawn_blocking,
time::{self, MissedTickBehavior},
};
use crates_index::{Crate, SparseIndex};
use crate::models::crates::CrateName;
#[derive(Clone)]
pub struct ManagedIndex {
index: Arc<Mutex<GitIndex>>,
index: Arc<SparseIndex>,
client: reqwest::Client,
}
impl ManagedIndex {
pub fn new() -> Self {
pub fn new(client: reqwest::Client) -> Self {
// the index path is configurable through the `CARGO_HOME` env variable
let index = Arc::new(Mutex::new(GitIndex::new_cargo_default().unwrap()));
let index = Arc::new(SparseIndex::new_cargo_default().unwrap());
Self { index }
Self { index, client }
}
pub fn crate_(&self, crate_name: CrateName) -> Option<Crate> {
self.index.lock().crate_(crate_name.as_ref())
}
/// Finds crate by name, returning a list of its versions.
///
/// # Errors
///
/// Returns error if HTTP request to crates index fails.
pub async fn crate_(&self, crate_name: &CrateName) -> anyhow::Result<Option<Crate>> {
let req = self.index.make_cache_request(crate_name.as_str())?;
let req = http_to_reqwest_req(req);
pub async fn refresh_at_interval(&self, update_interval: Duration) {
let mut update_interval = time::interval(update_interval);
update_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
let res = self.client.execute(req).await?;
let res = reqwest_to_http_res(res).await?;
loop {
if let Err(err) = self.refresh().await {
tracing::error!(
"failed refreshing the crates.io-index, the operation will be retried: {}",
error_reporter::Report::new(err),
);
}
update_interval.tick().await;
Ok(self
.index
.parse_cache_response(crate_name.as_str(), res, true)?)
}
}
async fn refresh(&self) -> Result<(), crates_index::Error> {
let index = Arc::clone(&self.index);
/// Converts an `http` request builder from `crates-index` to a `reqwest` request.
fn http_to_reqwest_req(req: http::request::Builder) -> reqwest::Request {
let req = req
.body(())
.expect("request from crates_index crate should be valid");
spawn_blocking(move || index.lock().update())
.await
.expect("blocking index update task should never panic")?;
let req = http::Request::from_parts(req.into_parts().0, Vec::new());
Ok(())
reqwest::Request::try_from(req).expect("request from crates_index crate should be valid")
}
/// Converts an `http` request builder from `crates-index` to a `reqwest` request.
///
/// # Errors
///
/// Returns error if reading HTTP response fails (e.g.: connection is lost while streaming payload).
async fn reqwest_to_http_res(res: reqwest::Response) -> anyhow::Result<http::Response<Vec<u8>>> {
use http_body_util::BodyExt as _;
let (res, body) = http::Response::from(res).into_parts();
let body = body.collect().await?.to_bytes().to_vec();
Ok(http::Response::from_parts(res, body))
}
#[cfg(test)]
mod tests {
use super::*;
static_assertions::assert_impl_all!(ManagedIndex: Send, Sync);
}