implement support for workspaces

src/engine/futures/crawl.rs

@@ -1,4 +1,5 @@
 use std::mem;
+use std::path::PathBuf;
 
 use failure::Error;
 use futures::{Async, Future, Poll, Stream};
@@ -14,12 +15,12 @@ pub struct CrawlManifestFuture {
     repo_path: RepoPath,
     engine: Engine,
     crawler: ManifestCrawler,
-    unordered: FuturesUnordered<Box<Future<Item=(String, String), Error=Error>>>
+    unordered: FuturesUnordered<Box<Future<Item=(PathBuf, String), Error=Error>>>
 }
 
 impl CrawlManifestFuture {
-    pub fn new(engine: &Engine, repo_path: RepoPath, entry_point: String) -> Self {
+    pub fn new(engine: &Engine, repo_path: RepoPath, entry_point: PathBuf) -> Self {
-        let future: Box<Future<Item=_, Error=_>> = Box::new(engine.retrieve_file_at_path(&repo_path, &entry_point)
+        let future: Box<Future<Item=_, Error=_>> = Box::new(engine.retrieve_manifest_at_path(&repo_path, &entry_point)
             .map(move |contents| (entry_point, contents)));
         let engine = engine.clone();
         let crawler = ManifestCrawler::new();
@@ -45,7 +46,7 @@ impl Future for CrawlManifestFuture {
             Some((path, raw_manifest)) => {
                 let output = self.crawler.step(path, raw_manifest)?;
                 for path in output.paths_of_interest.into_iter() {
-                    let future: Box<Future<Item=_, Error=_>> = Box::new(self.engine.retrieve_file_at_path(&self.repo_path, &path)
+                    let future: Box<Future<Item=_, Error=_>> = Box::new(self.engine.retrieve_manifest_at_path(&self.repo_path, &path)
                         .map(move |contents| (path, contents)));
                     self.unordered.push(future);
                 }

src/engine/machines/crawler.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::path::PathBuf;
 
 use failure::Error;
 
@@ -10,11 +11,11 @@ pub struct ManifestCrawlerOutput {
 }
 
 pub struct ManifestCrawlerStepOutput {
-    pub paths_of_interest: Vec<String>
+    pub paths_of_interest: Vec<PathBuf>
 }
 
 pub struct ManifestCrawler {
-    manifests: HashMap<String, CrateManifest>,
+    manifests: HashMap<PathBuf, CrateManifest>,
     leaf_crates: Vec<(CrateName, CrateDeps)>
 }
 
@@ -26,17 +27,32 @@ impl ManifestCrawler {
         }
     }
 
-    pub fn step(&mut self, path: String, raw_manifest: String) -> Result<ManifestCrawlerStepOutput, Error> {
+    pub fn step(&mut self, path: PathBuf, raw_manifest: String) -> Result<ManifestCrawlerStepOutput, Error> {
         let manifest = parse_manifest_toml(&raw_manifest)?;
-        self.manifests.insert(path, manifest.clone());
+        self.manifests.insert(path.clone(), manifest.clone());
+
+        let mut output = ManifestCrawlerStepOutput {
+            paths_of_interest: vec![]
+        };
+
         match manifest {
-            CrateManifest::Crate(name, deps) => {
+            CrateManifest::Package(name, deps) => {
                 self.leaf_crates.push((name, deps));
+            },
+            CrateManifest::Workspace { members } => {
+                for mut member in members {
+                    output.paths_of_interest.push(path.clone().join(member));
+                }
+            },
+            CrateManifest::Mixed { name, deps, members } => {
+                self.leaf_crates.push((name, deps));
+                for mut member in members {
+                    output.paths_of_interest.push(path.clone().join(member));
+                }
             }
         }
-        Ok(ManifestCrawlerStepOutput {
+
-            paths_of_interest: vec![]
+        Ok(output)
-        })
     }
 
     pub fn finalize(self) -> ManifestCrawlerOutput {
@@ -45,3 +61,123 @@ impl ManifestCrawler {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use semver::VersionReq;
+
+    use super::ManifestCrawler;
+
+    #[test]
+    fn simple_package_manifest() {
+        let manifest = r#"
+[package]
+name = "simpleton"
+"#;
+        let mut crawler = ManifestCrawler::new();
+        let step_output = crawler.step("Cargo.toml".into(), manifest.to_string()).unwrap();
+        assert_eq!(step_output.paths_of_interest.len(), 0);
+        let output = crawler.finalize();
+        assert_eq!(output.crates.len(), 1);
+        assert_eq!(output.crates[0].0.as_ref(), "simpleton");
+        assert_eq!(output.crates[0].1.main.len(), 0);
+        assert_eq!(output.crates[0].1.dev.len(), 0);
+        assert_eq!(output.crates[0].1.build.len(), 0);
+    }
+
+    #[test]
+    fn more_complex_package_manifest() {
+        let manifest = r#"
+[package]
+name = "more-complex"
+[dependencies]
+foo = "0.30.0"
+bar = { version = "1.2.0", optional = true }
+[dev-dependencies]
+quickcheck = "0.5"
+[build-dependencies]
+codegen = "0.0.1"
+"#;
+        let mut crawler = ManifestCrawler::new();
+        let step_output = crawler.step("/Cargo.toml".into(), manifest.to_string()).unwrap();
+        assert_eq!(step_output.paths_of_interest.len(), 0);
+        let output = crawler.finalize();
+        assert_eq!(output.crates.len(), 1);
+        assert_eq!(output.crates[0].0.as_ref(), "more-complex");
+        assert_eq!(output.crates[0].1.main.len(), 2);
+        assert_eq!(output.crates[0].1.main.get("foo").unwrap(),
+            &VersionReq::parse("0.30.0").unwrap());
+        assert_eq!(output.crates[0].1.main.get("bar").unwrap(),
+            &VersionReq::parse("1.2.0").unwrap());
+        assert_eq!(output.crates[0].1.dev.len(), 1);
+        assert_eq!(output.crates[0].1.dev.get("quickcheck").unwrap(),
+            &VersionReq::parse("0.5").unwrap());
+        assert_eq!(output.crates[0].1.build.len(), 1);
+        assert_eq!(output.crates[0].1.build.get("codegen").unwrap(),
+            &VersionReq::parse("0.0.1").unwrap());
+    }
+
+    #[test]
+    fn simple_workspace_manifest() {
+        let manifest = r#"
+[workspace]
+members = [
+  "lib/",
+  "codegen/",
+  "contrib/",
+]
+"#;
+        let mut crawler = ManifestCrawler::new();
+        let step_output = crawler.step("/".into(), manifest.to_string()).unwrap();
+        assert_eq!(step_output.paths_of_interest.len(), 3);
+        assert_eq!(step_output.paths_of_interest[0].to_str().unwrap(), "/lib/");
+        assert_eq!(step_output.paths_of_interest[1].to_str().unwrap(), "/codegen/");
+        assert_eq!(step_output.paths_of_interest[2].to_str().unwrap(), "/contrib/");
+    }
+
+    #[test]
+    fn mixed_package_and_workspace_manifest() {
+        let futures_manifest = r#"
+[package]
+name = "futures"
+
+[dependencies]
+
+[workspace]
+members = ["futures-cpupool"]
+"#;
+
+        let futures_cpupool_manifest = r#"
+[package]
+name = "futures-cpupool"
+
+[dependencies]
+num_cpus = "1.0"
+
+[dependencies.futures]
+path = ".."
+version = "0.1"
+default-features = false
+features = ["use_std"]
+"#;
+
+        let mut crawler = ManifestCrawler::new();
+        let step_output = crawler.step("/".into(), futures_manifest.to_string()).unwrap();
+        assert_eq!(step_output.paths_of_interest.len(), 1);
+        assert_eq!(step_output.paths_of_interest[0].to_str().unwrap(), "/futures-cpupool");
+        let step_output = crawler.step("/futures-cpupool".into(), futures_cpupool_manifest.to_string()).unwrap();
+        assert_eq!(step_output.paths_of_interest.len(), 0);
+        let output = crawler.finalize();
+        assert_eq!(output.crates.len(), 2);
+        assert_eq!(output.crates[0].0.as_ref(), "futures");
+        assert_eq!(output.crates[0].1.main.len(), 0);
+        assert_eq!(output.crates[0].1.dev.len(), 0);
+        assert_eq!(output.crates[0].1.build.len(), 0);
+        assert_eq!(output.crates[1].0.as_ref(), "futures-cpupool");
+        assert_eq!(output.crates[1].1.main.len(), 1);
+        assert_eq!(output.crates[1].1.main.get("num_cpus").unwrap(),
+            &VersionReq::parse("1.0").unwrap());
+        assert_eq!(output.crates[1].1.dev.len(), 0);
+        assert_eq!(output.crates[1].1.build.len(), 0);
+    }
+}

src/engine/mod.rs

@@ -1,3 +1,4 @@
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -64,7 +65,8 @@ impl Engine {
     pub fn analyze_dependencies(&self, repo_path: RepoPath) ->
         impl Future<Item=AnalyzeDependenciesOutcome, Error=Error>
     {
-        let manifest_future = CrawlManifestFuture::new(self, repo_path, "Cargo.toml".to_string());
+        let entry_point = PathBuf::from("/");
+        let manifest_future = CrawlManifestFuture::new(self, repo_path, entry_point);
 
         let engine = self.clone();
         manifest_future.and_then(move |manifest_output| {
@@ -88,9 +90,9 @@ impl Engine {
         })
     }
 
-    fn retrieve_file_at_path(&self, repo_path: &RepoPath, path: &str) ->
+    fn retrieve_manifest_at_path<P: AsRef<Path>>(&self, repo_path: &RepoPath, path: &P) ->
         impl Future<Item=String, Error=Error>
     {
-        retrieve_file_at_path(self.client.clone(), &repo_path, path).from_err()
+        retrieve_file_at_path(self.client.clone(), &repo_path, &path.as_ref().join("Cargo.toml")).from_err()
     }
 }

src/interactors/github.rs

@@ -1,3 +1,5 @@
+use std::path::Path;
+
 use failure::Error;
 use futures::{Future, IntoFuture, Stream, future};
 use hyper::{Error as HyperError, Method, Request, Response};
@@ -10,15 +12,16 @@ use ::models::repo::{Repository, RepoPath};
 const GITHUB_API_BASE_URI: &'static str = "https://api.github.com";
 const GITHUB_USER_CONTENT_BASE_URI: &'static str = "https://raw.githubusercontent.com";
 
-pub fn retrieve_file_at_path<S>(service: S, repo_path: &RepoPath, file_path: &str) ->
+pub fn retrieve_file_at_path<S, P: AsRef<Path>>(service: S, repo_path: &RepoPath, path: &P) ->
     impl Future<Item=String, Error=Error>
     where S: Service<Request=Request, Response=Response, Error=HyperError>
 {
+    let path_str = path.as_ref().to_str().expect("failed to convert path to str");
     let uri_future = format!("{}/{}/{}/master/{}",
         GITHUB_USER_CONTENT_BASE_URI,
         repo_path.qual.as_ref(),
         repo_path.name.as_ref(),
-        file_path
+        path_str
     ).parse().into_future().from_err();
 
     uri_future.and_then(move |uri| {

src/models/crates.rs

@@ -1,5 +1,6 @@
 use std::borrow::Borrow;
 use std::collections::BTreeMap;
+use std::path::PathBuf;
 use std::str::FromStr;
 
 use failure::Error;
@@ -111,5 +112,7 @@ impl AnalyzedDependencies {
 
 #[derive(Clone, Debug)]
 pub enum CrateManifest {
-    Crate(CrateName, CrateDeps)
+    Package(CrateName, CrateDeps),
+    Workspace { members: Vec<PathBuf> },
+    Mixed { name: CrateName, deps: CrateDeps, members: Vec<PathBuf> }
 }

src/parsers/manifest.rs

@@ -1,4 +1,5 @@
 use std::collections::BTreeMap;
+use std::path::PathBuf;
 
 use failure::Error;
 use semver::VersionReq;
@@ -25,9 +26,17 @@ struct CargoTomlPackage {
     name: String
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+struct CargoTomlWorkspace {
+    members: Vec<PathBuf>
+}
+
 #[derive(Serialize, Deserialize, Debug)]
 struct CargoToml {
-    package: CargoTomlPackage,
+    #[serde(default)]
+    package: Option<CargoTomlPackage>,
+    #[serde(default)]
+    workspace: Option<CargoTomlWorkspace>,
     #[serde(default)]
     dependencies: BTreeMap<String, CargoTomlDependency>,
     #[serde(rename = "dev-dependencies")]
@@ -64,20 +73,40 @@ fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option<Result
 pub fn parse_manifest_toml(input: &str) -> Result<CrateManifest, Error> {
     let cargo_toml = toml::de::from_str::<CargoToml>(input)?;
 
-    let crate_name = cargo_toml.package.name.parse::<CrateName>()?;
+    let mut package_part = None;
+    let mut workspace_part = None;
 
-    let dependencies = cargo_toml.dependencies
+    if let Some(package) =  cargo_toml.package {
-        .into_iter().filter_map(convert_dependency).collect::<Result<BTreeMap<_, _>, _>>()?;
+        let crate_name = package.name.parse::<CrateName>()?;
-    let dev_dependencies = cargo_toml.dev_dependencies
-        .into_iter().filter_map(convert_dependency).collect::<Result<BTreeMap<_, _>, _>>()?;
-    let build_dependencies = cargo_toml.build_dependencies
-        .into_iter().filter_map(convert_dependency).collect::<Result<BTreeMap<_, _>, _>>()?;
 
-    let deps = CrateDeps {
+        let dependencies = cargo_toml.dependencies
-        main: dependencies,
+            .into_iter().filter_map(convert_dependency).collect::<Result<BTreeMap<_, _>, _>>()?;
-        dev: dev_dependencies,
+        let dev_dependencies = cargo_toml.dev_dependencies
-        build: build_dependencies
+            .into_iter().filter_map(convert_dependency).collect::<Result<BTreeMap<_, _>, _>>()?;
-    };
+        let build_dependencies = cargo_toml.build_dependencies
+            .into_iter().filter_map(convert_dependency).collect::<Result<BTreeMap<_, _>, _>>()?;
 
-    Ok(CrateManifest::Crate(crate_name, deps))
+        let deps = CrateDeps {
+            main: dependencies,
+            dev: dev_dependencies,
+            build: build_dependencies
+        };
+
+        package_part = Some((crate_name, deps));
+    }
+
+    if let Some(workspace) = cargo_toml.workspace {
+        workspace_part = Some(workspace.members);
+    }
+
+    match (package_part, workspace_part) {
+        (Some((name, deps)), None) =>
+            Ok(CrateManifest::Package(name, deps)),
+        (None, Some(members)) =>
+            Ok(CrateManifest::Workspace { members }),
+        (Some((name, deps)), Some(members)) =>
+            Ok(CrateManifest::Mixed { name, deps, members }),
+        (None, None) =>
+            Err(format_err!("neither workspace nor package found in manifest"))
+    }
 }