diff --git a/src/engine/futures/analyze.rs b/src/engine/futures/analyze.rs index b9f4faf..61563b5 100644 --- a/src/engine/futures/analyze.rs +++ b/src/engine/futures/analyze.rs @@ -1,13 +1,12 @@ use failure::Error; -use futures::{Future, Poll, Stream, stream}; +use futures::{Future, Poll, Stream}; +use futures::stream::futures_unordered; use ::models::crates::{AnalyzedDependencies, CrateDeps}; use super::super::Engine; use super::super::machines::analyzer::DependencyAnalyzer; -const FETCH_RELEASES_CONCURRENCY: usize = 10; - pub struct AnalyzeDependenciesFuture { inner: Box> } @@ -28,8 +27,7 @@ impl AnalyzeDependenciesFuture { let release_futures = engine.fetch_releases(main_deps.chain(dev_deps).chain(build_deps)); - let analyzed_deps_future = stream::iter_ok::<_, Error>(release_futures) - .buffer_unordered(FETCH_RELEASES_CONCURRENCY) + let analyzed_deps_future = futures_unordered(release_futures) .fold(analyzer, |mut analyzer, releases| { analyzer.process(releases); Ok(analyzer) as Result<_, Error> }) .map(|analyzer| analyzer.finalize()); diff --git a/src/interactors/crates.rs b/src/interactors/crates.rs index c333f0f..496ffab 100644 --- a/src/interactors/crates.rs +++ b/src/interactors/crates.rs @@ -1,3 +1,5 @@ +use std::str; + use failure::Error; use futures::{Future, Stream, IntoFuture, future}; use hyper::{Error as HyperError, Method, Request, Response, Uri}; @@ -7,25 +9,21 @@ use serde_json; use ::models::crates::{CrateName, CrateRelease}; -const CRATES_API_BASE_URI: &'static str = "https://crates.io/api/v1"; +const CRATES_INDEX_BASE_URI: &str = "https://raw.githubusercontent.com/rust-lang/crates.io-index"; -#[derive(Serialize, Deserialize, Debug)] -struct CratesVersion { - num: Version, +#[derive(Deserialize, Debug)] +struct RegistryPackage { + vers: Version, + #[serde(default)] yanked: bool } -#[derive(Serialize, Deserialize, Debug)] -struct QueryCratesVersionsBody { - versions: Vec -} - -fn convert_body(name: &CrateName, body: QueryCratesVersionsBody) -> Result { - let releases = body.versions.into_iter().map(|version| { +fn convert_pkgs(name: &CrateName, packages: Vec) -> Result { + let releases = packages.into_iter().map(|package| { CrateRelease { name: name.clone(), - version: version.num, - yanked: version.yanked + version: package.vers, + yanked: package.yanked } }).collect(); @@ -42,7 +40,16 @@ pub fn query_crate(service: S, crate_name: CrateName) -> impl Future where S: Service { - let uri_future = format!("{}/crates/{}/versions", CRATES_API_BASE_URI, crate_name.as_ref()) + let lower_name = crate_name.as_ref().to_lowercase(); + + let path = match lower_name.len() { + 1 => format!("1/{}", lower_name), + 2 => format!("2/{}", lower_name), + 3 => format!("3/{}/{}", &lower_name[..1], lower_name), + _ => format!("{}/{}/{}", &lower_name[0..2], &lower_name[2..4], lower_name), + }; + + let uri_future = format!("{}/master/{}", CRATES_INDEX_BASE_URI, path) .parse::().into_future().from_err(); uri_future.and_then(move |uri| { @@ -55,10 +62,15 @@ pub fn query_crate(service: S, crate_name: CrateName) -> } else { let body_future = response.body().concat2().from_err(); let decode_future = body_future.and_then(|body| { - serde_json::from_slice::(&body) - .map_err(|err| err.into()) - }); - let convert_future = decode_future.and_then(move |body| convert_body(&crate_name, body)); + let string_body = str::from_utf8(body.as_ref())?; + let packages = string_body.lines() + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| serde_json::from_str::(s)) + .collect::>()?; + Ok(packages) + }); + let convert_future = decode_future.and_then(move |pkgs| convert_pkgs(&crate_name, pkgs)); future::Either::B(convert_future) } })