From 52e3fc4d283a3df38d99877b5e82ba557597a5d4 Mon Sep 17 00:00:00 2001 From: Sam Rijs Date: Sun, 11 Feb 2018 20:53:10 +1100 Subject: [PATCH] support path-based internal dependencies --- Cargo.toml | 1 + src/engine/futures/analyze.rs | 12 +++-- src/engine/futures/crawl.rs | 6 +-- src/engine/machines/analyzer.rs | 11 +++-- src/engine/machines/crawler.rs | 87 ++++++++++++++++++++------------- src/engine/mod.rs | 9 ++-- src/interactors/crates.rs | 8 +-- src/interactors/github.rs | 9 ++-- src/main.rs | 1 + src/models/crates.rs | 52 +++++++++++++++----- src/parsers/manifest.rs | 22 ++++++--- src/server/views/html/status.rs | 2 +- 12 files changed, 142 insertions(+), 78 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d804dbc..f231fc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ hyper-tls = "0.1.2" lazy_static = "1.0.0" maud = "0.17.2" ordermap = "0.4.0" +relative-path = { version = "0.3.7", features = ["serde"] } route-recognizer = "0.1.12" semver = { version = "0.9.0", features = ["serde"] } serde = "1.0.27" diff --git a/src/engine/futures/analyze.rs b/src/engine/futures/analyze.rs index 726707a..b9f4faf 100644 --- a/src/engine/futures/analyze.rs +++ b/src/engine/futures/analyze.rs @@ -16,9 +16,15 @@ impl AnalyzeDependenciesFuture { pub fn new(engine: &Engine, deps: CrateDeps) -> Self { let analyzer = DependencyAnalyzer::new(&deps); - let main_deps = deps.main.into_iter().map(|(name, _)| name); - let dev_deps = deps.dev.into_iter().map(|(name, _)| name); - let build_deps = deps.build.into_iter().map(|(name, _)| name); + let main_deps = deps.main.into_iter().filter_map(|(name, dep)| { + if dep.is_external() { Some(name) } else { None } + }); + let dev_deps = deps.dev.into_iter().filter_map(|(name, dep)| { + if dep.is_external() { Some(name) } else { None } + }); + let build_deps = deps.build.into_iter().filter_map(|(name, dep)| { + if dep.is_external() { Some(name) } else { None } + }); let release_futures = engine.fetch_releases(main_deps.chain(dev_deps).chain(build_deps)); diff --git a/src/engine/futures/crawl.rs b/src/engine/futures/crawl.rs index 8616dd9..beb199e 100644 --- a/src/engine/futures/crawl.rs +++ b/src/engine/futures/crawl.rs @@ -1,9 +1,9 @@ use std::mem; -use std::path::PathBuf; use failure::Error; use futures::{Async, Future, Poll, Stream}; use futures::stream::FuturesOrdered; +use relative_path::RelativePathBuf; use ::models::repo::RepoPath; @@ -15,11 +15,11 @@ pub struct CrawlManifestFuture { repo_path: RepoPath, engine: Engine, crawler: ManifestCrawler, - futures: FuturesOrdered>> + futures: FuturesOrdered>> } impl CrawlManifestFuture { - pub fn new(engine: &Engine, repo_path: RepoPath, entry_point: PathBuf) -> Self { + pub fn new(engine: &Engine, repo_path: RepoPath, entry_point: RelativePathBuf) -> Self { let future: Box> = Box::new(engine.retrieve_manifest_at_path(&repo_path, &entry_point) .map(move |contents| (entry_point, contents))); let engine = engine.clone(); diff --git a/src/engine/machines/analyzer.rs b/src/engine/machines/analyzer.rs index e593397..4628750 100644 --- a/src/engine/machines/analyzer.rs +++ b/src/engine/machines/analyzer.rs @@ -55,12 +55,13 @@ impl DependencyAnalyzer { #[cfg(test)] mod tests { - use super::{CrateDeps, CrateRelease, DependencyAnalyzer}; + use models::crates::{CrateDep, CrateDeps, CrateRelease}; + use super::DependencyAnalyzer; #[test] fn tracks_latest_without_matching() { let mut deps = CrateDeps::default(); - deps.main.insert("hyper".parse().unwrap(), "^0.11.0".parse().unwrap()); + deps.main.insert("hyper".parse().unwrap(), CrateDep::External("^0.11.0".parse().unwrap())); let mut analyzer = DependencyAnalyzer::new(&deps); analyzer.process(vec![ @@ -77,7 +78,7 @@ mod tests { #[test] fn tracks_latest_that_matches() { let mut deps = CrateDeps::default(); - deps.main.insert("hyper".parse().unwrap(), "^0.10.0".parse().unwrap()); + deps.main.insert("hyper".parse().unwrap(), CrateDep::External("^0.10.0".parse().unwrap())); let mut analyzer = DependencyAnalyzer::new(&deps); analyzer.process(vec![ @@ -95,7 +96,7 @@ mod tests { #[test] fn skips_yanked_releases() { let mut deps = CrateDeps::default(); - deps.main.insert("hyper".parse().unwrap(), "^0.10.0".parse().unwrap()); + deps.main.insert("hyper".parse().unwrap(), CrateDep::External("^0.10.0".parse().unwrap())); let mut analyzer = DependencyAnalyzer::new(&deps); analyzer.process(vec![ @@ -112,7 +113,7 @@ mod tests { #[test] fn skips_prereleases() { let mut deps = CrateDeps::default(); - deps.main.insert("hyper".parse().unwrap(), "^0.10.0".parse().unwrap()); + deps.main.insert("hyper".parse().unwrap(), CrateDep::External("^0.10.0".parse().unwrap())); let mut analyzer = DependencyAnalyzer::new(&deps); analyzer.process(vec![ diff --git a/src/engine/machines/crawler.rs b/src/engine/machines/crawler.rs index 8058fed..72119cb 100644 --- a/src/engine/machines/crawler.rs +++ b/src/engine/machines/crawler.rs @@ -1,22 +1,22 @@ use std::collections::HashMap; -use std::path::PathBuf; use failure::Error; +use relative_path::RelativePathBuf; use ordermap::map::OrderMap; use ::parsers::manifest::parse_manifest_toml; -use ::models::crates::{CrateDeps, CrateName, CrateManifest}; +use ::models::crates::{CrateDep, CrateDeps, CrateName, CrateManifest}; pub struct ManifestCrawlerOutput { pub crates: OrderMap } pub struct ManifestCrawlerStepOutput { - pub paths_of_interest: Vec + pub paths_of_interest: Vec } pub struct ManifestCrawler { - manifests: HashMap, + manifests: HashMap, leaf_crates: OrderMap } @@ -28,7 +28,7 @@ impl ManifestCrawler { } } - pub fn step(&mut self, path: PathBuf, raw_manifest: String) -> Result { + pub fn step(&mut self, path: RelativePathBuf, raw_manifest: String) -> Result { let manifest = parse_manifest_toml(&raw_manifest)?; self.manifests.insert(path.clone(), manifest.clone()); @@ -38,28 +38,45 @@ impl ManifestCrawler { match manifest { CrateManifest::Package(name, deps) => { - self.leaf_crates.insert(name, deps); + self.process_package(&path, name, deps, &mut output); }, CrateManifest::Workspace { members } => { - for mut member in members { - if !member.ends_with("*") { - output.paths_of_interest.push(path.clone().join(member)); - } - } + self.process_workspace(&path, &members, &mut output); }, CrateManifest::Mixed { name, deps, members } => { - self.leaf_crates.insert(name, deps); - for mut member in members { - if !member.ends_with("*") { - output.paths_of_interest.push(path.clone().join(member)); - } - } + self.process_package(&path, name, deps, &mut output); + self.process_workspace(&path, &members, &mut output); } } Ok(output) } + fn register_interest(&mut self, base_path: &RelativePathBuf, path: &RelativePathBuf, output: &mut ManifestCrawlerStepOutput) { + let full_path = base_path.join_normalized(path); + if !self.manifests.contains_key(&full_path) { + output.paths_of_interest.push(full_path); + } + } + + fn process_package(&mut self, base_path: &RelativePathBuf, name: CrateName, deps: CrateDeps, output: &mut ManifestCrawlerStepOutput) { + for (_, dep) in deps.main.iter().chain(deps.dev.iter()).chain(deps.build.iter()) { + if let &CrateDep::Internal(ref path) = dep { + self.register_interest(base_path, path, output); + } + } + + self.leaf_crates.insert(name, deps); + } + + fn process_workspace(&mut self, base_path: &RelativePathBuf, members: &[RelativePathBuf], output: &mut ManifestCrawlerStepOutput) { + for mut path in members { + if !path.ends_with("*") { + self.register_interest(base_path, path, output); + } + } + } + pub fn finalize(self) -> ManifestCrawlerOutput { ManifestCrawlerOutput { crates: self.leaf_crates @@ -69,8 +86,10 @@ impl ManifestCrawler { #[cfg(test)] mod tests { + use relative_path::RelativePath; use semver::VersionReq; + use models::crates::CrateDep; use super::ManifestCrawler; #[test] @@ -103,21 +122,21 @@ quickcheck = "0.5" codegen = "0.0.1" "#; let mut crawler = ManifestCrawler::new(); - let step_output = crawler.step("/Cargo.toml".into(), manifest.to_string()).unwrap(); + let step_output = crawler.step("".into(), manifest.to_string()).unwrap(); assert_eq!(step_output.paths_of_interest.len(), 0); let output = crawler.finalize(); assert_eq!(output.crates.len(), 1); assert_eq!(output.crates["more-complex"].main.len(), 2); assert_eq!(output.crates["more-complex"].main.get("foo").unwrap(), - &VersionReq::parse("0.30.0").unwrap()); + &CrateDep::External(VersionReq::parse("0.30.0").unwrap())); assert_eq!(output.crates["more-complex"].main.get("bar").unwrap(), - &VersionReq::parse("1.2.0").unwrap()); + &CrateDep::External(VersionReq::parse("1.2.0").unwrap())); assert_eq!(output.crates["more-complex"].dev.len(), 1); assert_eq!(output.crates["more-complex"].dev.get("quickcheck").unwrap(), - &VersionReq::parse("0.5").unwrap()); + &CrateDep::External(VersionReq::parse("0.5").unwrap())); assert_eq!(output.crates["more-complex"].build.len(), 1); assert_eq!(output.crates["more-complex"].build.get("codegen").unwrap(), - &VersionReq::parse("0.0.1").unwrap()); + &CrateDep::External(VersionReq::parse("0.0.1").unwrap())); } #[test] @@ -131,11 +150,11 @@ members = [ ] "#; let mut crawler = ManifestCrawler::new(); - let step_output = crawler.step("/".into(), manifest.to_string()).unwrap(); + let step_output = crawler.step("".into(), manifest.to_string()).unwrap(); assert_eq!(step_output.paths_of_interest.len(), 3); - assert_eq!(step_output.paths_of_interest[0].to_str().unwrap(), "/lib/"); - assert_eq!(step_output.paths_of_interest[1].to_str().unwrap(), "/codegen/"); - assert_eq!(step_output.paths_of_interest[2].to_str().unwrap(), "/contrib/"); + assert_eq!(step_output.paths_of_interest[0].as_str(), "lib"); + assert_eq!(step_output.paths_of_interest[1].as_str(), "codegen"); + assert_eq!(step_output.paths_of_interest[2].as_str(), "contrib"); } #[test] @@ -148,9 +167,9 @@ members = [ ] "#; let mut crawler = ManifestCrawler::new(); - let step_output = crawler.step("/".into(), manifest.to_string()).unwrap(); + let step_output = crawler.step("".into(), manifest.to_string()).unwrap(); assert_eq!(step_output.paths_of_interest.len(), 1); - assert_eq!(step_output.paths_of_interest[0].to_str().unwrap(), "/lib/"); + assert_eq!(step_output.paths_of_interest[0].as_str(), "lib"); } #[test] @@ -180,19 +199,21 @@ features = ["use_std"] "#; let mut crawler = ManifestCrawler::new(); - let step_output = crawler.step("/".into(), futures_manifest.to_string()).unwrap(); + let step_output = crawler.step("".into(), futures_manifest.to_string()).unwrap(); assert_eq!(step_output.paths_of_interest.len(), 1); - assert_eq!(step_output.paths_of_interest[0].to_str().unwrap(), "/futures-cpupool"); - let step_output = crawler.step("/futures-cpupool".into(), futures_cpupool_manifest.to_string()).unwrap(); + assert_eq!(step_output.paths_of_interest[0].as_str(), "futures-cpupool"); + let step_output = crawler.step("futures-cpupool".into(), futures_cpupool_manifest.to_string()).unwrap(); assert_eq!(step_output.paths_of_interest.len(), 0); let output = crawler.finalize(); assert_eq!(output.crates.len(), 2); assert_eq!(output.crates["futures"].main.len(), 0); assert_eq!(output.crates["futures"].dev.len(), 0); assert_eq!(output.crates["futures"].build.len(), 0); - assert_eq!(output.crates["futures-cpupool"].main.len(), 1); + assert_eq!(output.crates["futures-cpupool"].main.len(), 2); assert_eq!(output.crates["futures-cpupool"].main.get("num_cpus").unwrap(), - &VersionReq::parse("1.0").unwrap()); + &CrateDep::External(VersionReq::parse("1.0").unwrap())); + assert_eq!(output.crates["futures-cpupool"].main.get("futures").unwrap(), + &CrateDep::Internal(RelativePath::new("..").to_relative_path_buf())); assert_eq!(output.crates["futures-cpupool"].dev.len(), 0); assert_eq!(output.crates["futures-cpupool"].build.len(), 0); } diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 0ed6c4a..74701b0 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -1,5 +1,4 @@ use std::collections::HashSet; -use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -9,6 +8,7 @@ use futures::future::join_all; use hyper::Client; use hyper::client::HttpConnector; use hyper_tls::HttpsConnector; +use relative_path::{RelativePath, RelativePathBuf}; use slog::Logger; use tokio_service::Service; @@ -70,7 +70,7 @@ impl Engine { pub fn analyze_dependencies(&self, repo_path: RepoPath) -> impl Future { - let entry_point = PathBuf::from("/"); + let entry_point = RelativePath::new("/").to_relative_path_buf(); let manifest_future = CrawlManifestFuture::new(self, repo_path, entry_point); let engine = self.clone(); @@ -95,10 +95,11 @@ impl Engine { }) } - fn retrieve_manifest_at_path>(&self, repo_path: &RepoPath, path: &P) -> + fn retrieve_manifest_at_path(&self, repo_path: &RepoPath, path: &RelativePathBuf) -> impl Future { - retrieve_file_at_path(self.client.clone(), &repo_path, &path.as_ref().join("Cargo.toml")).from_err() + let manifest_path = path.join(RelativePath::new("Cargo.toml")); + retrieve_file_at_path(self.client.clone(), &repo_path, &manifest_path).from_err() } } diff --git a/src/interactors/crates.rs b/src/interactors/crates.rs index fcadd83..c333f0f 100644 --- a/src/interactors/crates.rs +++ b/src/interactors/crates.rs @@ -1,6 +1,6 @@ use failure::Error; use futures::{Future, Stream, IntoFuture, future}; -use hyper::{Error as HyperError, Method, Request, Response}; +use hyper::{Error as HyperError, Method, Request, Response, Uri}; use tokio_service::Service; use semver::Version; use serde_json; @@ -43,15 +43,15 @@ pub fn query_crate(service: S, crate_name: CrateName) -> where S: Service { let uri_future = format!("{}/crates/{}/versions", CRATES_API_BASE_URI, crate_name.as_ref()) - .parse().into_future().from_err(); + .parse::().into_future().from_err(); uri_future.and_then(move |uri| { - let request = Request::new(Method::Get, uri); + let request = Request::new(Method::Get, uri.clone()); service.call(request).from_err().and_then(move |response| { let status = response.status(); if !status.is_success() { - future::Either::A(future::err(format_err!("Status code: {}", status))) + future::Either::A(future::err(format_err!("Status code {} for URI {}", status, uri))) } else { let body_future = response.body().concat2().from_err(); let decode_future = body_future.and_then(|body| { diff --git a/src/interactors/github.rs b/src/interactors/github.rs index fd4027e..a3e95cd 100644 --- a/src/interactors/github.rs +++ b/src/interactors/github.rs @@ -1,9 +1,8 @@ -use std::path::Path; - use failure::Error; use futures::{Future, IntoFuture, Stream, future}; use hyper::{Error as HyperError, Method, Request, Response, Uri}; use hyper::header::UserAgent; +use relative_path::RelativePathBuf; use tokio_service::Service; use serde_json; @@ -12,11 +11,11 @@ use ::models::repo::{Repository, RepoPath}; const GITHUB_API_BASE_URI: &'static str = "https://api.github.com"; const GITHUB_USER_CONTENT_BASE_URI: &'static str = "https://raw.githubusercontent.com"; -pub fn retrieve_file_at_path>(service: S, repo_path: &RepoPath, path: &P) -> +pub fn retrieve_file_at_path(service: S, repo_path: &RepoPath, path: &RelativePathBuf) -> impl Future where S: Service { - let path_str = path.as_ref().to_str().expect("failed to convert path to str"); + let path_str: &str = path.as_ref(); let uri_future = format!("{}/{}/{}/master/{}", GITHUB_USER_CONTENT_BASE_URI, repo_path.qual.as_ref(), @@ -83,7 +82,7 @@ impl Service for GetPopularRepos service.call(request).from_err().and_then(|response| { let status = response.status(); if !status.is_success() { - future::Either::A(future::err(format_err!("Status code: {}", status))) + future::Either::A(future::err(format_err!("Status code {} for popular repo search", status))) } else { let body_future = response.body().concat2().from_err(); let decode_future = body_future diff --git a/src/main.rs b/src/main.rs index 4176812..95483b5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,6 +11,7 @@ extern crate hyper_tls; #[macro_use] extern crate lazy_static; extern crate maud; extern crate ordermap; +extern crate relative_path; extern crate route_recognizer; extern crate semver; #[macro_use] extern crate serde_derive; diff --git a/src/models/crates.rs b/src/models/crates.rs index 5f8827a..07f11b7 100644 --- a/src/models/crates.rs +++ b/src/models/crates.rs @@ -1,9 +1,9 @@ use std::borrow::Borrow; use std::collections::BTreeMap; -use std::path::PathBuf; use std::str::FromStr; use failure::Error; +use relative_path::RelativePathBuf; use semver::{Version, VersionReq}; #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -50,11 +50,27 @@ pub struct CrateRelease { pub yanked: bool } +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CrateDep { + External(VersionReq), + Internal(RelativePathBuf) +} + +impl CrateDep { + pub fn is_external(&self) -> bool { + if let &CrateDep::External(_) = self { + true + } else { + false + } + } +} + #[derive(Clone, Debug, Default)] pub struct CrateDeps { - pub main: BTreeMap, - pub dev: BTreeMap, - pub build: BTreeMap + pub main: BTreeMap, + pub dev: BTreeMap, + pub build: BTreeMap } #[derive(Debug)] @@ -87,14 +103,26 @@ pub struct AnalyzedDependencies { impl AnalyzedDependencies { pub fn new(deps: &CrateDeps) -> AnalyzedDependencies { - let main = deps.main.iter().map(|(name, req)| { - (name.clone(), AnalyzedDependency::new(req.clone())) + let main = deps.main.iter().filter_map(|(name, dep)| { + if let &CrateDep::External(ref req) = dep { + Some((name.clone(), AnalyzedDependency::new(req.clone()))) + } else { + None + } }).collect(); - let dev = deps.dev.iter().map(|(name, req)| { - (name.clone(), AnalyzedDependency::new(req.clone())) + let dev = deps.dev.iter().filter_map(|(name, dep)| { + if let &CrateDep::External(ref req) = dep { + Some((name.clone(), AnalyzedDependency::new(req.clone()))) + } else { + None + } }).collect(); - let build = deps.build.iter().map(|(name, req)| { - (name.clone(), AnalyzedDependency::new(req.clone())) + let build = deps.build.iter().filter_map(|(name, dep)| { + if let &CrateDep::External(ref req) = dep { + Some((name.clone(), AnalyzedDependency::new(req.clone()))) + } else { + None + } }).collect(); AnalyzedDependencies { main, dev, build } } @@ -113,6 +141,6 @@ impl AnalyzedDependencies { #[derive(Clone, Debug)] pub enum CrateManifest { Package(CrateName, CrateDeps), - Workspace { members: Vec }, - Mixed { name: CrateName, deps: CrateDeps, members: Vec } + Workspace { members: Vec }, + Mixed { name: CrateName, deps: CrateDeps, members: Vec } } diff --git a/src/parsers/manifest.rs b/src/parsers/manifest.rs index 8b9370b..0d4fda5 100644 --- a/src/parsers/manifest.rs +++ b/src/parsers/manifest.rs @@ -1,16 +1,16 @@ use std::collections::BTreeMap; -use std::path::PathBuf; use failure::Error; +use relative_path::RelativePathBuf; use semver::VersionReq; use toml; -use ::models::crates::{CrateName, CrateDeps, CrateManifest}; +use ::models::crates::{CrateName, CrateDep, CrateDeps, CrateManifest}; #[derive(Serialize, Deserialize, Debug)] struct CargoTomlComplexDependency { git: Option, - path: Option, + path: Option, version: Option } @@ -28,7 +28,7 @@ struct CargoTomlPackage { #[derive(Serialize, Deserialize, Debug)] struct CargoTomlWorkspace { - members: Vec + members: Vec } #[derive(Serialize, Deserialize, Debug)] @@ -47,22 +47,28 @@ struct CargoToml { build_dependencies: BTreeMap } -fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option> { +fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option> { match cargo_dep { (name, CargoTomlDependency::Simple(string)) => { Some(name.parse::().map_err(|err| err.into()).and_then(|parsed_name| { string.parse::().map_err(|err| err.into()) - .map(|version| (parsed_name, version)) + .map(|version| (parsed_name, CrateDep::External(version))) })) } (name, CargoTomlDependency::Complex(cplx)) => { - if cplx.git.is_some() || cplx.path.is_some() { + if cplx.git.is_some() { None + } else if cplx.path.is_some() { + cplx.path.map(|path| { + name.parse::().map_err(|err| err.into()).map(|parsed_name| { + (parsed_name, CrateDep::Internal(path)) + }) + }) } else { cplx.version.map(|string| { name.parse::().map_err(|err| err.into()).and_then(|parsed_name| { string.parse::().map_err(|err| err.into()) - .map(|version| (parsed_name, version)) + .map(|version| (parsed_name, CrateDep::External(version))) }) }) } diff --git a/src/server/views/html/status.rs b/src/server/views/html/status.rs index 25bc460..8ac8b54 100644 --- a/src/server/views/html/status.rs +++ b/src/server/views/html/status.rs @@ -17,7 +17,7 @@ fn dependency_tables(crate_name: CrateName, deps: AnalyzedDependencies) -> Marku } @if deps.main.is_empty() && deps.dev.is_empty() && deps.build.is_empty() { - p class="notification has-text-centered" "No dependencies! 🙌" + p class="notification has-text-centered" "No external dependencies! 🙌" } @if !deps.main.is_empty() {