From e1c921066bd0f371907d2babdc45470921f36fe4 Mon Sep 17 00:00:00 2001 From: Sam Rijs Date: Sun, 11 Feb 2018 14:46:58 +1100 Subject: [PATCH] implement support for workspaces --- src/engine/futures/crawl.rs | 9 +- src/engine/machines/crawler.rs | 152 +++++++++++++++++++++++++++++++-- src/engine/mod.rs | 8 +- src/interactors/github.rs | 7 +- src/models/crates.rs | 5 +- src/parsers/manifest.rs | 57 ++++++++++--- 6 files changed, 206 insertions(+), 32 deletions(-) diff --git a/src/engine/futures/crawl.rs b/src/engine/futures/crawl.rs index 3918dcb..c671b13 100644 --- a/src/engine/futures/crawl.rs +++ b/src/engine/futures/crawl.rs @@ -1,4 +1,5 @@ use std::mem; +use std::path::PathBuf; use failure::Error; use futures::{Async, Future, Poll, Stream}; @@ -14,12 +15,12 @@ pub struct CrawlManifestFuture { repo_path: RepoPath, engine: Engine, crawler: ManifestCrawler, - unordered: FuturesUnordered>> + unordered: FuturesUnordered>> } impl CrawlManifestFuture { - pub fn new(engine: &Engine, repo_path: RepoPath, entry_point: String) -> Self { - let future: Box> = Box::new(engine.retrieve_file_at_path(&repo_path, &entry_point) + pub fn new(engine: &Engine, repo_path: RepoPath, entry_point: PathBuf) -> Self { + let future: Box> = Box::new(engine.retrieve_manifest_at_path(&repo_path, &entry_point) .map(move |contents| (entry_point, contents))); let engine = engine.clone(); let crawler = ManifestCrawler::new(); @@ -45,7 +46,7 @@ impl Future for CrawlManifestFuture { Some((path, raw_manifest)) => { let output = self.crawler.step(path, raw_manifest)?; for path in output.paths_of_interest.into_iter() { - let future: Box> = Box::new(self.engine.retrieve_file_at_path(&self.repo_path, &path) + let future: Box> = Box::new(self.engine.retrieve_manifest_at_path(&self.repo_path, &path) .map(move |contents| (path, contents))); self.unordered.push(future); } diff --git a/src/engine/machines/crawler.rs b/src/engine/machines/crawler.rs index 4fbbd85..1f9492e 100644 --- a/src/engine/machines/crawler.rs +++ b/src/engine/machines/crawler.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::path::PathBuf; use failure::Error; @@ -10,11 +11,11 @@ pub struct ManifestCrawlerOutput { } pub struct ManifestCrawlerStepOutput { - pub paths_of_interest: Vec + pub paths_of_interest: Vec } pub struct ManifestCrawler { - manifests: HashMap, + manifests: HashMap, leaf_crates: Vec<(CrateName, CrateDeps)> } @@ -26,17 +27,32 @@ impl ManifestCrawler { } } - pub fn step(&mut self, path: String, raw_manifest: String) -> Result { + pub fn step(&mut self, path: PathBuf, raw_manifest: String) -> Result { let manifest = parse_manifest_toml(&raw_manifest)?; - self.manifests.insert(path, manifest.clone()); + self.manifests.insert(path.clone(), manifest.clone()); + + let mut output = ManifestCrawlerStepOutput { + paths_of_interest: vec![] + }; + match manifest { - CrateManifest::Crate(name, deps) => { + CrateManifest::Package(name, deps) => { self.leaf_crates.push((name, deps)); + }, + CrateManifest::Workspace { members } => { + for mut member in members { + output.paths_of_interest.push(path.clone().join(member)); + } + }, + CrateManifest::Mixed { name, deps, members } => { + self.leaf_crates.push((name, deps)); + for mut member in members { + output.paths_of_interest.push(path.clone().join(member)); + } } } - Ok(ManifestCrawlerStepOutput { - paths_of_interest: vec![] - }) + + Ok(output) } pub fn finalize(self) -> ManifestCrawlerOutput { @@ -45,3 +61,123 @@ impl ManifestCrawler { } } } + +#[cfg(test)] +mod tests { + use semver::VersionReq; + + use super::ManifestCrawler; + + #[test] + fn simple_package_manifest() { + let manifest = r#" +[package] +name = "simpleton" +"#; + let mut crawler = ManifestCrawler::new(); + let step_output = crawler.step("Cargo.toml".into(), manifest.to_string()).unwrap(); + assert_eq!(step_output.paths_of_interest.len(), 0); + let output = crawler.finalize(); + assert_eq!(output.crates.len(), 1); + assert_eq!(output.crates[0].0.as_ref(), "simpleton"); + assert_eq!(output.crates[0].1.main.len(), 0); + assert_eq!(output.crates[0].1.dev.len(), 0); + assert_eq!(output.crates[0].1.build.len(), 0); + } + + #[test] + fn more_complex_package_manifest() { + let manifest = r#" +[package] +name = "more-complex" +[dependencies] +foo = "0.30.0" +bar = { version = "1.2.0", optional = true } +[dev-dependencies] +quickcheck = "0.5" +[build-dependencies] +codegen = "0.0.1" +"#; + let mut crawler = ManifestCrawler::new(); + let step_output = crawler.step("/Cargo.toml".into(), manifest.to_string()).unwrap(); + assert_eq!(step_output.paths_of_interest.len(), 0); + let output = crawler.finalize(); + assert_eq!(output.crates.len(), 1); + assert_eq!(output.crates[0].0.as_ref(), "more-complex"); + assert_eq!(output.crates[0].1.main.len(), 2); + assert_eq!(output.crates[0].1.main.get("foo").unwrap(), + &VersionReq::parse("0.30.0").unwrap()); + assert_eq!(output.crates[0].1.main.get("bar").unwrap(), + &VersionReq::parse("1.2.0").unwrap()); + assert_eq!(output.crates[0].1.dev.len(), 1); + assert_eq!(output.crates[0].1.dev.get("quickcheck").unwrap(), + &VersionReq::parse("0.5").unwrap()); + assert_eq!(output.crates[0].1.build.len(), 1); + assert_eq!(output.crates[0].1.build.get("codegen").unwrap(), + &VersionReq::parse("0.0.1").unwrap()); + } + + #[test] + fn simple_workspace_manifest() { + let manifest = r#" +[workspace] +members = [ + "lib/", + "codegen/", + "contrib/", +] +"#; + let mut crawler = ManifestCrawler::new(); + let step_output = crawler.step("/".into(), manifest.to_string()).unwrap(); + assert_eq!(step_output.paths_of_interest.len(), 3); + assert_eq!(step_output.paths_of_interest[0].to_str().unwrap(), "/lib/"); + assert_eq!(step_output.paths_of_interest[1].to_str().unwrap(), "/codegen/"); + assert_eq!(step_output.paths_of_interest[2].to_str().unwrap(), "/contrib/"); + } + + #[test] + fn mixed_package_and_workspace_manifest() { + let futures_manifest = r#" +[package] +name = "futures" + +[dependencies] + +[workspace] +members = ["futures-cpupool"] +"#; + + let futures_cpupool_manifest = r#" +[package] +name = "futures-cpupool" + +[dependencies] +num_cpus = "1.0" + +[dependencies.futures] +path = ".." +version = "0.1" +default-features = false +features = ["use_std"] +"#; + + let mut crawler = ManifestCrawler::new(); + let step_output = crawler.step("/".into(), futures_manifest.to_string()).unwrap(); + assert_eq!(step_output.paths_of_interest.len(), 1); + assert_eq!(step_output.paths_of_interest[0].to_str().unwrap(), "/futures-cpupool"); + let step_output = crawler.step("/futures-cpupool".into(), futures_cpupool_manifest.to_string()).unwrap(); + assert_eq!(step_output.paths_of_interest.len(), 0); + let output = crawler.finalize(); + assert_eq!(output.crates.len(), 2); + assert_eq!(output.crates[0].0.as_ref(), "futures"); + assert_eq!(output.crates[0].1.main.len(), 0); + assert_eq!(output.crates[0].1.dev.len(), 0); + assert_eq!(output.crates[0].1.build.len(), 0); + assert_eq!(output.crates[1].0.as_ref(), "futures-cpupool"); + assert_eq!(output.crates[1].1.main.len(), 1); + assert_eq!(output.crates[1].1.main.get("num_cpus").unwrap(), + &VersionReq::parse("1.0").unwrap()); + assert_eq!(output.crates[1].1.dev.len(), 0); + assert_eq!(output.crates[1].1.build.len(), 0); + } +} diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 476c1fa..2bd682d 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -1,3 +1,4 @@ +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -64,7 +65,8 @@ impl Engine { pub fn analyze_dependencies(&self, repo_path: RepoPath) -> impl Future { - let manifest_future = CrawlManifestFuture::new(self, repo_path, "Cargo.toml".to_string()); + let entry_point = PathBuf::from("/"); + let manifest_future = CrawlManifestFuture::new(self, repo_path, entry_point); let engine = self.clone(); manifest_future.and_then(move |manifest_output| { @@ -88,9 +90,9 @@ impl Engine { }) } - fn retrieve_file_at_path(&self, repo_path: &RepoPath, path: &str) -> + fn retrieve_manifest_at_path>(&self, repo_path: &RepoPath, path: &P) -> impl Future { - retrieve_file_at_path(self.client.clone(), &repo_path, path).from_err() + retrieve_file_at_path(self.client.clone(), &repo_path, &path.as_ref().join("Cargo.toml")).from_err() } } diff --git a/src/interactors/github.rs b/src/interactors/github.rs index cd359e6..b87d0f5 100644 --- a/src/interactors/github.rs +++ b/src/interactors/github.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use failure::Error; use futures::{Future, IntoFuture, Stream, future}; use hyper::{Error as HyperError, Method, Request, Response}; @@ -10,15 +12,16 @@ use ::models::repo::{Repository, RepoPath}; const GITHUB_API_BASE_URI: &'static str = "https://api.github.com"; const GITHUB_USER_CONTENT_BASE_URI: &'static str = "https://raw.githubusercontent.com"; -pub fn retrieve_file_at_path(service: S, repo_path: &RepoPath, file_path: &str) -> +pub fn retrieve_file_at_path>(service: S, repo_path: &RepoPath, path: &P) -> impl Future where S: Service { + let path_str = path.as_ref().to_str().expect("failed to convert path to str"); let uri_future = format!("{}/{}/{}/master/{}", GITHUB_USER_CONTENT_BASE_URI, repo_path.qual.as_ref(), repo_path.name.as_ref(), - file_path + path_str ).parse().into_future().from_err(); uri_future.and_then(move |uri| { diff --git a/src/models/crates.rs b/src/models/crates.rs index 7aaac09..78ef916 100644 --- a/src/models/crates.rs +++ b/src/models/crates.rs @@ -1,5 +1,6 @@ use std::borrow::Borrow; use std::collections::BTreeMap; +use std::path::PathBuf; use std::str::FromStr; use failure::Error; @@ -111,5 +112,7 @@ impl AnalyzedDependencies { #[derive(Clone, Debug)] pub enum CrateManifest { - Crate(CrateName, CrateDeps) + Package(CrateName, CrateDeps), + Workspace { members: Vec }, + Mixed { name: CrateName, deps: CrateDeps, members: Vec } } diff --git a/src/parsers/manifest.rs b/src/parsers/manifest.rs index 545c7ec..8b9370b 100644 --- a/src/parsers/manifest.rs +++ b/src/parsers/manifest.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::path::PathBuf; use failure::Error; use semver::VersionReq; @@ -25,9 +26,17 @@ struct CargoTomlPackage { name: String } +#[derive(Serialize, Deserialize, Debug)] +struct CargoTomlWorkspace { + members: Vec +} + #[derive(Serialize, Deserialize, Debug)] struct CargoToml { - package: CargoTomlPackage, + #[serde(default)] + package: Option, + #[serde(default)] + workspace: Option, #[serde(default)] dependencies: BTreeMap, #[serde(rename = "dev-dependencies")] @@ -64,20 +73,40 @@ fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option Result { let cargo_toml = toml::de::from_str::(input)?; - let crate_name = cargo_toml.package.name.parse::()?; + let mut package_part = None; + let mut workspace_part = None; - let dependencies = cargo_toml.dependencies - .into_iter().filter_map(convert_dependency).collect::, _>>()?; - let dev_dependencies = cargo_toml.dev_dependencies - .into_iter().filter_map(convert_dependency).collect::, _>>()?; - let build_dependencies = cargo_toml.build_dependencies - .into_iter().filter_map(convert_dependency).collect::, _>>()?; + if let Some(package) = cargo_toml.package { + let crate_name = package.name.parse::()?; - let deps = CrateDeps { - main: dependencies, - dev: dev_dependencies, - build: build_dependencies - }; + let dependencies = cargo_toml.dependencies + .into_iter().filter_map(convert_dependency).collect::, _>>()?; + let dev_dependencies = cargo_toml.dev_dependencies + .into_iter().filter_map(convert_dependency).collect::, _>>()?; + let build_dependencies = cargo_toml.build_dependencies + .into_iter().filter_map(convert_dependency).collect::, _>>()?; - Ok(CrateManifest::Crate(crate_name, deps)) + let deps = CrateDeps { + main: dependencies, + dev: dev_dependencies, + build: build_dependencies + }; + + package_part = Some((crate_name, deps)); + } + + if let Some(workspace) = cargo_toml.workspace { + workspace_part = Some(workspace.members); + } + + match (package_part, workspace_part) { + (Some((name, deps)), None) => + Ok(CrateManifest::Package(name, deps)), + (None, Some(members)) => + Ok(CrateManifest::Workspace { members }), + (Some((name, deps)), Some(members)) => + Ok(CrateManifest::Mixed { name, deps, members }), + (None, None) => + Err(format_err!("neither workspace nor package found in manifest")) + } }