diff --git a/Cargo.toml b/Cargo.toml index 9b7ac73..4d84773 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ authors = ["Sam Rijs "] futures = "0.1.18" hyper = "0.11.15" hyper-tls = "0.1.2" +semver = { version = "0.9.0", features = ["serde"] } serde = "1.0.27" serde_derive = "1.0.27" serde_json = "1.0.9" @@ -14,3 +15,4 @@ slog = "2.1.1" slog-json = "2.2.0" tokio-core = "0.1.12" tokio-service = "0.1.0" +toml = "0.4.5" diff --git a/src/api.rs b/src/api.rs new file mode 100644 index 0000000..dfe4c1d --- /dev/null +++ b/src/api.rs @@ -0,0 +1,84 @@ +use std::collections::BTreeMap; + +use futures::{Future, future}; +use hyper::{Client, Error as HyperError, Request, Response, StatusCode}; +use hyper::client::HttpConnector; +use hyper::header::ContentType; +use hyper_tls::HttpsConnector; +use semver::{Version, VersionReq}; +use serde_json; +use slog::Logger; +use tokio_service::Service; + +use ::models::repo::RepoPath; +use ::engine::Engine; + +pub struct Api { + pub engine: Engine +} + +#[derive(Debug, Serialize)] +struct AnalyzeDependenciesResponseDetail { + required: VersionReq, + latest: Option, + outdated: bool +} + +#[derive(Debug, Serialize)] +struct AnalyzeDependenciesResponse { + dependencies: BTreeMap, + #[serde(rename="dev-dependencies")] + dev_dependencies: BTreeMap, + #[serde(rename="build-dependencies")] + build_dependencies: BTreeMap +} + +impl Service for Api { + type Request = Request; + type Response = Response; + type Error = HyperError; + type Future = Box>; + + fn call(&self, req: Request) -> Self::Future { + let repo_path = RepoPath::from_parts("github.com", "hyperium", "hyper").unwrap(); + + let future = self.engine.analyze_dependencies(repo_path).then(|result| { + match result { + Err(err) => { + let mut response = Response::new(); + response.set_status(StatusCode::InternalServerError); + response.set_body(format!("{:?}", err)); + future::Either::A(future::ok(response)) + }, + Ok(dependencies) => { + let response_struct = AnalyzeDependenciesResponse { + dependencies: dependencies.main.into_iter() + .map(|(name, analyzed)| (name.into(), AnalyzeDependenciesResponseDetail { + outdated: analyzed.is_outdated(), + required: analyzed.required, + latest: analyzed.latest + })).collect(), + dev_dependencies: dependencies.dev.into_iter() + .map(|(name, analyzed)| (name.into(), AnalyzeDependenciesResponseDetail { + outdated: analyzed.is_outdated(), + required: analyzed.required, + latest: analyzed.latest + })).collect(), + build_dependencies: dependencies.build.into_iter() + .map(|(name, analyzed)| (name.into(), AnalyzeDependenciesResponseDetail { + outdated: analyzed.is_outdated(), + required: analyzed.required, + latest: analyzed.latest + })).collect() + }; + let mut response = Response::new() + .with_header(ContentType::json()) + .with_body(serde_json::to_string(&response_struct).unwrap()); + future::Either::B(future::ok(response)) + } + } + }); + + Box::new(future) + } +} diff --git a/src/engine/analyzer.rs b/src/engine/analyzer.rs new file mode 100644 index 0000000..2135fe1 --- /dev/null +++ b/src/engine/analyzer.rs @@ -0,0 +1,55 @@ +use std::collections::BTreeMap; +use std::collections::btree_map::{Entry as BTreeMapEntry}; + +use semver::{Version, VersionReq}; + +use ::models::crates::{CrateName, CrateDeps, CrateRelease, AnalyzedDependency, AnalyzedDependencies}; + +pub struct DependencyAnalyzer { + deps: AnalyzedDependencies +} + +impl DependencyAnalyzer { + pub fn new(deps: &CrateDeps) -> DependencyAnalyzer { + DependencyAnalyzer { + deps: AnalyzedDependencies::new(deps) + } + } + + fn process_single(dep: &mut AnalyzedDependency, ver: &Version) { + if dep.required.matches(&ver) { + if let Some(ref mut current_latest_that_matches) = dep.latest_that_matches { + if *current_latest_that_matches < *ver { + *current_latest_that_matches = ver.clone(); + } + } else { + dep.latest_that_matches = Some(ver.clone()); + } + } + if let Some(ref mut current_latest) = dep.latest { + if *current_latest < *ver { + *current_latest = ver.clone(); + } + } else { + dep.latest = Some(ver.clone()); + } + } + + pub fn process>(&mut self, releases: I) { + for release in releases.into_iter().filter(|r| !r.yanked) { + if let Some(main_dep) = self.deps.main.get_mut(&release.name) { + DependencyAnalyzer::process_single(main_dep, &release.version) + } + if let Some(dev_dep) = self.deps.dev.get_mut(&release.name) { + DependencyAnalyzer::process_single(dev_dep, &release.version) + } + if let Some(build_dep) = self.deps.build.get_mut(&release.name) { + DependencyAnalyzer::process_single(build_dep, &release.version) + } + } + } + + pub fn finalize(self) -> AnalyzedDependencies { + self.deps + } +} diff --git a/src/engine/mod.rs b/src/engine/mod.rs new file mode 100644 index 0000000..3dce9ca --- /dev/null +++ b/src/engine/mod.rs @@ -0,0 +1,79 @@ +mod analyzer; + +use futures::{Future, Stream, future, stream}; +use hyper::{Client, Error as HyperError, Request, Response, StatusCode}; +use hyper::client::HttpConnector; +use hyper_tls::HttpsConnector; +use slog::Logger; + +use ::models::repo::RepoPath; +use ::models::crates::{CrateName, CrateRelease, CrateManifest, AnalyzedDependencies}; + +use ::parsers::manifest::{ManifestParseError, parse_manifest_toml}; + +use ::interactors::crates::{QueryCrateError, query_crate}; +use ::interactors::github::{RetrieveFileAtPathError, retrieve_file_at_path}; + +use self::analyzer::DependencyAnalyzer; + +#[derive(Clone, Debug)] +pub struct Engine { + pub client: Client>, + pub logger: Logger +} + +#[derive(Debug)] +pub enum AnalyzeDependenciesError { + QueryCrate(QueryCrateError), + RetrieveFileAtPath(RetrieveFileAtPathError), + ParseManifest(ManifestParseError) +} + +const FETCH_RELEASES_CONCURRENCY: usize = 10; + +impl Engine { + pub fn analyze_dependencies(&self, repo_path: RepoPath) -> + impl Future + { + let manifest_future = self.retrieve_manifest(&repo_path); + + let engine = self.clone(); + manifest_future.and_then(move |manifest| { + let CrateManifest::Crate(deps) = manifest; + let analyzer = DependencyAnalyzer::new(&deps); + + let main_deps = deps.main.into_iter().map(|(name, _)| name); + let dev_deps = deps.dev.into_iter().map(|(name, _)| name); + let build_deps = deps.build.into_iter().map(|(name, _)| name); + + let release_futures = engine.fetch_releases(main_deps.chain(dev_deps).chain(build_deps)); + + stream::iter_ok(release_futures) + .buffer_unordered(FETCH_RELEASES_CONCURRENCY) + .fold(analyzer, |mut analyzer, releases| { analyzer.process(releases); Ok(analyzer) }) + .map(|analyzer| analyzer.finalize()) + }) + } + + fn fetch_releases>(&self, names: I) -> + impl Iterator, Error=AnalyzeDependenciesError>> + { + let client = self.client.clone(); + names.into_iter().map(move |name| { + query_crate(client.clone(), name) + .map_err(AnalyzeDependenciesError::QueryCrate) + .map(|resp| resp.releases) + }) + } + + fn retrieve_manifest(&self, repo_path: &RepoPath) -> + impl Future + { + retrieve_file_at_path(self.client.clone(), &repo_path, "Cargo.toml") + .map_err(AnalyzeDependenciesError::RetrieveFileAtPath) + .and_then(|manifest_source| { + parse_manifest_toml(&manifest_source) + .map_err(AnalyzeDependenciesError::ParseManifest) + }) + } +} diff --git a/src/interactors/crates.rs b/src/interactors/crates.rs new file mode 100644 index 0000000..5347b65 --- /dev/null +++ b/src/interactors/crates.rs @@ -0,0 +1,74 @@ +use futures::{Future, Stream, IntoFuture, future}; +use hyper::{Error as HyperError, Method, Request, Response, StatusCode}; +use hyper::error::UriError; +use tokio_service::Service; +use semver::Version; +use serde_json; + +use ::models::crates::{CrateName, CrateRelease}; + +const CRATES_API_BASE_URI: &'static str = "https://crates.io/api/v1"; + +#[derive(Serialize, Deserialize, Debug)] +struct CratesVersion { + num: Version, + yanked: bool +} + +#[derive(Serialize, Deserialize, Debug)] +struct QueryCratesVersionsBody { + versions: Vec +} + +fn convert_body(name: &CrateName, body: QueryCratesVersionsBody) -> Result { + let releases = body.versions.into_iter().map(|version| { + CrateRelease { + name: name.clone(), + version: version.num, + yanked: version.yanked + } + }).collect(); + + Ok(QueryCrateResponse { + releases: releases + }) +} + +pub struct QueryCrateResponse { + pub releases: Vec +} + +#[derive(Debug)] +pub enum QueryCrateError { + Uri(UriError), + Status(StatusCode), + Transport(HyperError), + Decode(serde_json::Error) +} + +pub fn query_crate(service: S, crate_name: CrateName) -> + impl Future + where S: Service +{ + let uri_future = format!("{}/crates/{}/versions", CRATES_API_BASE_URI, crate_name.as_ref()) + .parse().into_future().map_err(QueryCrateError::Uri); + + uri_future.and_then(move |uri| { + let request = Request::new(Method::Get, uri); + + service.call(request).map_err(QueryCrateError::Transport).and_then(move |response| { + let status = response.status(); + if !status.is_success() { + future::Either::A(future::err(QueryCrateError::Status(status))) + } else { + let body_future = response.body().concat2().map_err(QueryCrateError::Transport); + let decode_future = body_future.and_then(|body| { + serde_json::from_slice::(&body) + .map_err(QueryCrateError::Decode) + }); + let convert_future = decode_future.and_then(move |body| convert_body(&crate_name, body)); + future::Either::B(convert_future) + } + }) + }) +} diff --git a/src/interactors/github.rs b/src/interactors/github.rs new file mode 100644 index 0000000..ea74e09 --- /dev/null +++ b/src/interactors/github.rs @@ -0,0 +1,46 @@ +use std::string::FromUtf8Error; + +use futures::{Future, IntoFuture, Stream, future}; +use hyper::{Error as HyperError, Method, Request, Response, StatusCode}; +use hyper::error::UriError; +use tokio_service::Service; + +use ::models::repo::RepoPath; + +const GITHUB_USER_CONTENT_BASE_URI: &'static str = "https://raw.githubusercontent.com"; + +#[derive(Debug)] +pub enum RetrieveFileAtPathError { + Uri(UriError), + Transport(HyperError), + Status(StatusCode), + Decode(FromUtf8Error) +} + +pub fn retrieve_file_at_path(service: S, repo_path: &RepoPath, file_path: &str) -> + impl Future + where S: Service +{ + let uri_future = format!("{}/{}/{}/master/{}", + GITHUB_USER_CONTENT_BASE_URI, + repo_path.qual.as_ref(), + repo_path.name.as_ref(), + file_path + ).parse().into_future().map_err(RetrieveFileAtPathError::Uri); + + uri_future.and_then(move |uri| { + let request = Request::new(Method::Get, uri); + + service.call(request).map_err(RetrieveFileAtPathError::Transport).and_then(|response| { + let status = response.status(); + if !status.is_success() { + future::Either::A(future::err(RetrieveFileAtPathError::Status(status))) + } else { + let body_future = response.body().concat2().map_err(RetrieveFileAtPathError::Transport); + let decode_future = body_future + .and_then(|body| String::from_utf8(body.to_vec()).map_err(RetrieveFileAtPathError::Decode)); + future::Either::B(decode_future) + } + }) + }) +} diff --git a/src/robots/mod.rs b/src/interactors/mod.rs similarity index 50% rename from src/robots/mod.rs rename to src/interactors/mod.rs index 30e78d1..bccf822 100644 --- a/src/robots/mod.rs +++ b/src/interactors/mod.rs @@ -1 +1,2 @@ pub mod crates; +pub mod github; diff --git a/src/main.rs b/src/main.rs index d526855..41fd86b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ extern crate futures; extern crate hyper; extern crate hyper_tls; +extern crate semver; #[macro_use] extern crate serde_derive; extern crate serde; extern crate serde_json; @@ -11,10 +12,13 @@ extern crate serde_json; extern crate slog_json; extern crate tokio_core; extern crate tokio_service; +extern crate toml; mod models; -mod robots; -mod serve; +mod parsers; +mod interactors; +mod engine; +mod api; use std::net::SocketAddr; use std::sync::Mutex; @@ -26,7 +30,8 @@ use hyper_tls::HttpsConnector; use slog::Drain; use tokio_core::reactor::Core; -use self::serve::Serve; +use self::api::Api; +use self::engine::Engine; fn main() { let logger = slog::Logger::root( @@ -51,12 +56,13 @@ fn main() { let http = Http::new(); - let serve_logger = logger.clone(); + let engine = Engine { + client: client.clone(), + logger: logger.clone() + }; + let serve = http.serve_addr_handle(&addr, &handle, move || { - Ok(Serve { - client: client.clone(), - logger: serve_logger.clone() - }) + Ok(Api { engine: engine.clone() }) }).expect("failed to bind server"); let serving = serve.for_each(move |conn| { diff --git a/src/models/crates.rs b/src/models/crates.rs index 0dd0de4..5466b69 100644 --- a/src/models/crates.rs +++ b/src/models/crates.rs @@ -1,14 +1,24 @@ +use std::collections::BTreeMap; use std::str::FromStr; +use semver::{Version, VersionReq}; + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct CrateName(String); +impl Into for CrateName { + fn into(self) -> String { + self.0 + } +} + #[derive(Debug)] pub struct CrateNameValidationError; impl AsRef for CrateName { - fn as_ref(&self) -> &str { - self.0.as_ref() - } + fn as_ref(&self) -> &str { + self.0.as_ref() + } } impl FromStr for CrateName { @@ -26,3 +36,72 @@ impl FromStr for CrateName { } } } + +#[derive(Debug)] +pub struct CrateRelease { + pub name: CrateName, + pub version: Version, + pub yanked: bool +} + +#[derive(Clone, Copy, Debug)] +pub enum DependencyType { + Main, + Dev, + Build +} + +#[derive(Debug)] +pub struct CrateDeps { + pub main: BTreeMap, + pub dev: BTreeMap, + pub build: BTreeMap +} + +#[derive(Debug)] +pub struct AnalyzedDependency { + pub required: VersionReq, + pub latest_that_matches: Option, + pub latest: Option +} + +impl AnalyzedDependency { + pub fn new(required: VersionReq) -> AnalyzedDependency { + AnalyzedDependency { + required, + latest_that_matches: None, + latest: None + } + } + + pub fn is_outdated(&self) -> bool { + self.latest > self.latest_that_matches + } +} + +#[derive(Debug)] +pub struct AnalyzedDependencies { + pub main: BTreeMap, + pub dev: BTreeMap, + pub build: BTreeMap +} + +impl AnalyzedDependencies { + pub fn new(deps: &CrateDeps) -> AnalyzedDependencies { + let main = deps.main.iter().map(|(name, req)| { + (name.clone(), AnalyzedDependency::new(req.clone())) + }).collect(); + let dev = deps.dev.iter().map(|(name, req)| { + (name.clone(), AnalyzedDependency::new(req.clone())) + }).collect(); + let build = deps.build.iter().map(|(name, req)| { + (name.clone(), AnalyzedDependency::new(req.clone())) + }).collect(); + AnalyzedDependencies { main, dev, build } + } +} + +#[derive(Debug)] +pub enum CrateManifest { + Crate(CrateDeps) +} diff --git a/src/models/mod.rs b/src/models/mod.rs index 30e78d1..290c408 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -1 +1,2 @@ pub mod crates; +pub mod repo; diff --git a/src/models/repo.rs b/src/models/repo.rs new file mode 100644 index 0000000..1727173 --- /dev/null +++ b/src/models/repo.rs @@ -0,0 +1,92 @@ +use std::str::FromStr; + +pub struct RepoPath { + pub site: RepoSite, + pub qual: RepoQualifier, + pub name: RepoName +} + +impl RepoPath { + pub fn from_parts(site: &str, qual: &str, name: &str) -> Result { + Ok(RepoPath { + site: site.parse()?, + qual: qual.parse()?, + name: name.parse()? + }) + } +} + +#[derive(Debug)] +pub struct RepoValidationError; + +pub struct RepoSite(String); + +impl FromStr for RepoSite { + type Err = RepoValidationError; + + fn from_str(input: &str) -> Result { + let is_valid = input.chars().all(|c| { + c.is_ascii_alphanumeric() || c == '.' + }); + + if !is_valid { + Err(RepoValidationError) + } else { + Ok(RepoSite(input.to_string())) + } + } +} + +impl AsRef for RepoSite { + fn as_ref(&self) -> &str { + self.0.as_ref() + } +} + +pub struct RepoQualifier(String); + +impl FromStr for RepoQualifier { + type Err = RepoValidationError; + + fn from_str(input: &str) -> Result { + let is_valid = input.chars().all(|c| { + c.is_ascii_alphanumeric() || c == '-' || c == '_' + }); + + if !is_valid { + Err(RepoValidationError) + } else { + Ok(RepoQualifier(input.to_string())) + } + } +} + +impl AsRef for RepoQualifier { + fn as_ref(&self) -> &str { + self.0.as_ref() + } +} + +pub struct RepoName(String); + +impl FromStr for RepoName { + type Err = RepoValidationError; + + fn from_str(input: &str) -> Result { + let is_valid = input.chars().all(|c| { + c.is_ascii_alphanumeric() || c == '-' || c == '_' + }); + + if !is_valid { + Err(RepoValidationError) + } else { + Ok(RepoName(input.to_string())) + } + } +} + +impl AsRef for RepoName { + fn as_ref(&self) -> &str { + self.0.as_ref() + } +} diff --git a/src/parsers/manifest.rs b/src/parsers/manifest.rs new file mode 100644 index 0000000..125402b --- /dev/null +++ b/src/parsers/manifest.rs @@ -0,0 +1,82 @@ +use std::collections::BTreeMap; + +use semver::{ReqParseError, VersionReq}; +use toml; + +use ::models::crates::{CrateName, CrateDeps, CrateManifest, CrateNameValidationError}; + +#[derive(Debug)] +pub enum ManifestParseError { + Serde(toml::de::Error), + Name(CrateNameValidationError), + Version(ReqParseError) +} + +#[derive(Serialize, Deserialize, Debug)] +struct CargoTomlComplexDependency { + git: Option, + path: Option, + version: Option +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(untagged)] +enum CargoTomlDependency { + Simple(String), + Complex(CargoTomlComplexDependency) +} + +#[derive(Serialize, Deserialize, Debug)] +struct CargoToml { + #[serde(default)] + dependencies: BTreeMap, + #[serde(rename = "dev-dependencies")] + #[serde(default)] + dev_dependencies: BTreeMap, + #[serde(rename = "build-dependencies")] + #[serde(default)] + build_dependencies: BTreeMap +} + +fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option> { + match cargo_dep { + (name, CargoTomlDependency::Simple(string)) => { + Some(name.parse().map_err(ManifestParseError::Name).and_then(|parsed_name| { + string.parse().map_err(ManifestParseError::Version) + .map(|version| (parsed_name, version)) + })) + } + (name, CargoTomlDependency::Complex(cplx)) => { + if cplx.git.is_some() || cplx.path.is_some() { + None + } else { + cplx.version.map(|string| { + name.parse().map_err(ManifestParseError::Name).and_then(|parsed_name| { + string.parse().map_err(ManifestParseError::Version) + .map(|version| (parsed_name, version)) + }) + }) + } + } + } +} + +pub fn parse_manifest_toml(input: &str) -> Result { + let cargo_toml = toml::de::from_str::(input) + .map_err(ManifestParseError::Serde)?; + + let dependencies = cargo_toml.dependencies + .into_iter().filter_map(convert_dependency).collect::, _>>()?; + let dev_dependencies = cargo_toml.dev_dependencies + .into_iter().filter_map(convert_dependency).collect::, _>>()?; + let build_dependencies = cargo_toml.build_dependencies + .into_iter().filter_map(convert_dependency).collect::, _>>()?; + + let deps = CrateDeps { + main: dependencies, + dev: dev_dependencies, + build: build_dependencies + }; + + Ok(CrateManifest::Crate(deps)) +} diff --git a/src/parsers/mod.rs b/src/parsers/mod.rs new file mode 100644 index 0000000..640fc64 --- /dev/null +++ b/src/parsers/mod.rs @@ -0,0 +1 @@ +pub mod manifest; diff --git a/src/robots/crates.rs b/src/robots/crates.rs deleted file mode 100644 index 3b6f452..0000000 --- a/src/robots/crates.rs +++ /dev/null @@ -1,52 +0,0 @@ -use futures::{Future, Stream, IntoFuture, future}; -use hyper::{Error as HyperError, Method, Request, Response, StatusCode}; -use hyper::error::UriError; -use tokio_service::Service; -use serde_json; - -use ::models::crates::CrateName; - -const CRATES_API_BASE_URI: &'static str = "https://crates.io/api/v1"; - -#[derive(Serialize, Deserialize, Debug)] -pub struct CratesVersion { - num: String, - yanked: bool -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct QueryCratesVersionsResponse { - versions: Vec -} - -#[derive(Debug)] -pub enum QueryCratesVersionsError { - Uri(UriError), - Status(StatusCode), - Transport(HyperError), - Decode(serde_json::Error) -} - -pub fn query_crates_versions(service: S, crate_name: &CrateName) -> - impl Future - where S: Service -{ - let uri_future = format!("{}/crates/{}/versions", CRATES_API_BASE_URI, crate_name.as_ref()) - .parse().into_future().map_err(QueryCratesVersionsError::Uri); - - uri_future.and_then(move |uri| { - let request = Request::new(Method::Get, uri); - - service.call(request).map_err(QueryCratesVersionsError::Transport).and_then(|response| { - let status = response.status(); - if !status.is_success() { - future::Either::A(future::err(QueryCratesVersionsError::Status(status))) - } else { - let body_future = response.body().concat2().map_err(QueryCratesVersionsError::Transport); - let decode_future = body_future - .and_then(|body| serde_json::from_slice(&body).map_err(QueryCratesVersionsError::Decode)); - future::Either::B(decode_future) - } - }) - }) -} diff --git a/src/robots/github.rs b/src/robots/github.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/serve.rs b/src/serve.rs deleted file mode 100644 index c827d23..0000000 --- a/src/serve.rs +++ /dev/null @@ -1,42 +0,0 @@ -use futures::{Future, future}; -use hyper::{Client, Error as HyperError, Request, Response, StatusCode}; -use hyper::client::HttpConnector; -use hyper_tls::HttpsConnector; -use slog::Logger; -use tokio_service::Service; - -use ::robots::crates::query_crates_versions; - -pub struct Serve { - pub client: Client>, - pub logger: Logger -} - -impl Service for Serve { - type Request = Request; - type Response = Response; - type Error = HyperError; - type Future = Box>; - - fn call(&self, req: Request) -> Self::Future { - let crate_name = "hyper".parse().unwrap(); - - let future = query_crates_versions(self.client.clone(), &crate_name).then(|result| { - match result { - Err(err) => { - let mut response = Response::new(); - response.set_status(StatusCode::InternalServerError); - response.set_body(format!("{:?}", err)); - future::Either::A(future::ok(response)) - }, - Ok(crates_response) => { - let mut response = Response::new(); - response.set_body(format!("{:?}", crates_response)); - future::Either::B(future::ok(response)) - } - } - }); - - Box::new(future) - } -}