support path-based internal dependencies

Cargo.toml

@@ -12,6 +12,7 @@ hyper-tls = "0.1.2"
 lazy_static = "1.0.0"
 maud = "0.17.2"
 ordermap = "0.4.0"
+relative-path = { version = "0.3.7", features = ["serde"] }
 route-recognizer = "0.1.12"
 semver = { version = "0.9.0", features = ["serde"] }
 serde = "1.0.27"

src/engine/futures/analyze.rs

@@ -16,9 +16,15 @@ impl AnalyzeDependenciesFuture {
     pub fn new(engine: &Engine, deps: CrateDeps) -> Self {
         let analyzer = DependencyAnalyzer::new(&deps);
 
-        let main_deps = deps.main.into_iter().map(|(name, _)| name);
+        let main_deps = deps.main.into_iter().filter_map(|(name, dep)| {
-        let dev_deps = deps.dev.into_iter().map(|(name, _)| name);
+            if dep.is_external() { Some(name) } else { None }
-        let build_deps = deps.build.into_iter().map(|(name, _)| name);
+        });
+        let dev_deps = deps.dev.into_iter().filter_map(|(name, dep)| {
+            if dep.is_external() { Some(name) } else { None }
+        });
+        let build_deps = deps.build.into_iter().filter_map(|(name, dep)| {
+            if dep.is_external() { Some(name) } else { None }
+        });
 
         let release_futures = engine.fetch_releases(main_deps.chain(dev_deps).chain(build_deps));
 

src/engine/futures/crawl.rs

@@ -1,9 +1,9 @@
 use std::mem;
-use std::path::PathBuf;
 
 use failure::Error;
 use futures::{Async, Future, Poll, Stream};
 use futures::stream::FuturesOrdered;
+use relative_path::RelativePathBuf;
 
 use ::models::repo::RepoPath;
 
@@ -15,11 +15,11 @@ pub struct CrawlManifestFuture {
     repo_path: RepoPath,
     engine: Engine,
     crawler: ManifestCrawler,
-    futures: FuturesOrdered<Box<Future<Item=(PathBuf, String), Error=Error>>>
+    futures: FuturesOrdered<Box<Future<Item=(RelativePathBuf, String), Error=Error>>>
 }
 
 impl CrawlManifestFuture {
-    pub fn new(engine: &Engine, repo_path: RepoPath, entry_point: PathBuf) -> Self {
+    pub fn new(engine: &Engine, repo_path: RepoPath, entry_point: RelativePathBuf) -> Self {
         let future: Box<Future<Item=_, Error=_>> = Box::new(engine.retrieve_manifest_at_path(&repo_path, &entry_point)
             .map(move |contents| (entry_point, contents)));
         let engine = engine.clone();

src/engine/machines/analyzer.rs

@@ -55,12 +55,13 @@ impl DependencyAnalyzer {
 
 #[cfg(test)]
 mod tests {
-    use super::{CrateDeps, CrateRelease, DependencyAnalyzer};
+    use models::crates::{CrateDep, CrateDeps, CrateRelease};
+    use super::DependencyAnalyzer;
 
     #[test]
     fn tracks_latest_without_matching() {
         let mut deps = CrateDeps::default();
-        deps.main.insert("hyper".parse().unwrap(), "^0.11.0".parse().unwrap());
+        deps.main.insert("hyper".parse().unwrap(), CrateDep::External("^0.11.0".parse().unwrap()));
 
         let mut analyzer = DependencyAnalyzer::new(&deps);
         analyzer.process(vec![
@@ -77,7 +78,7 @@ mod tests {
     #[test]
     fn tracks_latest_that_matches() {
         let mut deps = CrateDeps::default();
-        deps.main.insert("hyper".parse().unwrap(), "^0.10.0".parse().unwrap());
+        deps.main.insert("hyper".parse().unwrap(), CrateDep::External("^0.10.0".parse().unwrap()));
 
         let mut analyzer = DependencyAnalyzer::new(&deps);
         analyzer.process(vec![
@@ -95,7 +96,7 @@ mod tests {
     #[test]
     fn skips_yanked_releases() {
         let mut deps = CrateDeps::default();
-        deps.main.insert("hyper".parse().unwrap(), "^0.10.0".parse().unwrap());
+        deps.main.insert("hyper".parse().unwrap(), CrateDep::External("^0.10.0".parse().unwrap()));
 
         let mut analyzer = DependencyAnalyzer::new(&deps);
         analyzer.process(vec![
@@ -112,7 +113,7 @@ mod tests {
     #[test]
     fn skips_prereleases() {
         let mut deps = CrateDeps::default();
-        deps.main.insert("hyper".parse().unwrap(), "^0.10.0".parse().unwrap());
+        deps.main.insert("hyper".parse().unwrap(), CrateDep::External("^0.10.0".parse().unwrap()));
 
         let mut analyzer = DependencyAnalyzer::new(&deps);
         analyzer.process(vec![

src/engine/machines/crawler.rs

@@ -1,22 +1,22 @@
 use std::collections::HashMap;
-use std::path::PathBuf;
 
 use failure::Error;
+use relative_path::RelativePathBuf;
 use ordermap::map::OrderMap;
 
 use ::parsers::manifest::parse_manifest_toml;
-use ::models::crates::{CrateDeps, CrateName, CrateManifest};
+use ::models::crates::{CrateDep, CrateDeps, CrateName, CrateManifest};
 
 pub struct ManifestCrawlerOutput {
     pub crates: OrderMap<CrateName, CrateDeps>
 }
 
 pub struct ManifestCrawlerStepOutput {
-    pub paths_of_interest: Vec<PathBuf>
+    pub paths_of_interest: Vec<RelativePathBuf>
 }
 
 pub struct ManifestCrawler {
-    manifests: HashMap<PathBuf, CrateManifest>,
+    manifests: HashMap<RelativePathBuf, CrateManifest>,
     leaf_crates: OrderMap<CrateName, CrateDeps>
 }
 
@@ -28,7 +28,7 @@ impl ManifestCrawler {
         }
     }
 
-    pub fn step(&mut self, path: PathBuf, raw_manifest: String) -> Result<ManifestCrawlerStepOutput, Error> {
+    pub fn step(&mut self, path: RelativePathBuf, raw_manifest: String) -> Result<ManifestCrawlerStepOutput, Error> {
         let manifest = parse_manifest_toml(&raw_manifest)?;
         self.manifests.insert(path.clone(), manifest.clone());
 
@@ -38,28 +38,45 @@ impl ManifestCrawler {
 
         match manifest {
             CrateManifest::Package(name, deps) => {
-                self.leaf_crates.insert(name, deps);
+                self.process_package(&path, name, deps, &mut output);
             },
             CrateManifest::Workspace { members } => {
-                for mut member in members {
+                self.process_workspace(&path, &members, &mut output);
-                    if !member.ends_with("*") {
-                        output.paths_of_interest.push(path.clone().join(member));
-                    }
-                }
             },
             CrateManifest::Mixed { name, deps, members } => {
-                self.leaf_crates.insert(name, deps);
+                self.process_package(&path, name, deps, &mut output);
-                for mut member in members {
+                self.process_workspace(&path, &members, &mut output);
-                    if !member.ends_with("*") {
-                        output.paths_of_interest.push(path.clone().join(member));
-                    }
-                }
             }
         }
 
         Ok(output)
     }
 
+    fn register_interest(&mut self, base_path: &RelativePathBuf, path: &RelativePathBuf, output: &mut ManifestCrawlerStepOutput) {
+        let full_path = base_path.join_normalized(path);
+        if !self.manifests.contains_key(&full_path) {
+            output.paths_of_interest.push(full_path);
+        }
+    }
+
+    fn process_package(&mut self, base_path: &RelativePathBuf, name: CrateName, deps: CrateDeps, output: &mut ManifestCrawlerStepOutput) {
+        for (_, dep) in deps.main.iter().chain(deps.dev.iter()).chain(deps.build.iter()) {
+            if let &CrateDep::Internal(ref path) = dep {
+                self.register_interest(base_path, path, output);
+            }
+        }
+
+        self.leaf_crates.insert(name, deps);
+    }
+
+    fn process_workspace(&mut self, base_path: &RelativePathBuf, members: &[RelativePathBuf], output: &mut ManifestCrawlerStepOutput) {
+        for mut path in members {
+            if !path.ends_with("*") {
+                self.register_interest(base_path, path, output);
+            }
+        }
+    }
+
     pub fn finalize(self) -> ManifestCrawlerOutput {
         ManifestCrawlerOutput {
             crates: self.leaf_crates
@@ -69,8 +86,10 @@ impl ManifestCrawler {
 
 #[cfg(test)]
 mod tests {
+    use relative_path::RelativePath;
     use semver::VersionReq;
 
+    use models::crates::CrateDep;
     use super::ManifestCrawler;
 
     #[test]
@@ -103,21 +122,21 @@ quickcheck = "0.5"
 codegen = "0.0.1"
 "#;
         let mut crawler = ManifestCrawler::new();
-        let step_output = crawler.step("/Cargo.toml".into(), manifest.to_string()).unwrap();
+        let step_output = crawler.step("".into(), manifest.to_string()).unwrap();
         assert_eq!(step_output.paths_of_interest.len(), 0);
         let output = crawler.finalize();
         assert_eq!(output.crates.len(), 1);
         assert_eq!(output.crates["more-complex"].main.len(), 2);
         assert_eq!(output.crates["more-complex"].main.get("foo").unwrap(),
-            &VersionReq::parse("0.30.0").unwrap());
+            &CrateDep::External(VersionReq::parse("0.30.0").unwrap()));
         assert_eq!(output.crates["more-complex"].main.get("bar").unwrap(),
-            &VersionReq::parse("1.2.0").unwrap());
+            &CrateDep::External(VersionReq::parse("1.2.0").unwrap()));
         assert_eq!(output.crates["more-complex"].dev.len(), 1);
         assert_eq!(output.crates["more-complex"].dev.get("quickcheck").unwrap(),
-            &VersionReq::parse("0.5").unwrap());
+            &CrateDep::External(VersionReq::parse("0.5").unwrap()));
         assert_eq!(output.crates["more-complex"].build.len(), 1);
         assert_eq!(output.crates["more-complex"].build.get("codegen").unwrap(),
-            &VersionReq::parse("0.0.1").unwrap());
+            &CrateDep::External(VersionReq::parse("0.0.1").unwrap()));
     }
 
     #[test]
@@ -131,11 +150,11 @@ members = [
 ]
 "#;
         let mut crawler = ManifestCrawler::new();
-        let step_output = crawler.step("/".into(), manifest.to_string()).unwrap();
+        let step_output = crawler.step("".into(), manifest.to_string()).unwrap();
         assert_eq!(step_output.paths_of_interest.len(), 3);
-        assert_eq!(step_output.paths_of_interest[0].to_str().unwrap(), "/lib/");
+        assert_eq!(step_output.paths_of_interest[0].as_str(), "lib");
-        assert_eq!(step_output.paths_of_interest[1].to_str().unwrap(), "/codegen/");
+        assert_eq!(step_output.paths_of_interest[1].as_str(), "codegen");
-        assert_eq!(step_output.paths_of_interest[2].to_str().unwrap(), "/contrib/");
+        assert_eq!(step_output.paths_of_interest[2].as_str(), "contrib");
     }
 
     #[test]
@@ -148,9 +167,9 @@ members = [
 ]
 "#;
         let mut crawler = ManifestCrawler::new();
-        let step_output = crawler.step("/".into(), manifest.to_string()).unwrap();
+        let step_output = crawler.step("".into(), manifest.to_string()).unwrap();
         assert_eq!(step_output.paths_of_interest.len(), 1);
-        assert_eq!(step_output.paths_of_interest[0].to_str().unwrap(), "/lib/");
+        assert_eq!(step_output.paths_of_interest[0].as_str(), "lib");
     }
 
     #[test]
@@ -180,19 +199,21 @@ features = ["use_std"]
 "#;
 
         let mut crawler = ManifestCrawler::new();
-        let step_output = crawler.step("/".into(), futures_manifest.to_string()).unwrap();
+        let step_output = crawler.step("".into(), futures_manifest.to_string()).unwrap();
         assert_eq!(step_output.paths_of_interest.len(), 1);
-        assert_eq!(step_output.paths_of_interest[0].to_str().unwrap(), "/futures-cpupool");
+        assert_eq!(step_output.paths_of_interest[0].as_str(), "futures-cpupool");
-        let step_output = crawler.step("/futures-cpupool".into(), futures_cpupool_manifest.to_string()).unwrap();
+        let step_output = crawler.step("futures-cpupool".into(), futures_cpupool_manifest.to_string()).unwrap();
         assert_eq!(step_output.paths_of_interest.len(), 0);
         let output = crawler.finalize();
         assert_eq!(output.crates.len(), 2);
         assert_eq!(output.crates["futures"].main.len(), 0);
         assert_eq!(output.crates["futures"].dev.len(), 0);
         assert_eq!(output.crates["futures"].build.len(), 0);
-        assert_eq!(output.crates["futures-cpupool"].main.len(), 1);
+        assert_eq!(output.crates["futures-cpupool"].main.len(), 2);
         assert_eq!(output.crates["futures-cpupool"].main.get("num_cpus").unwrap(),
-            &VersionReq::parse("1.0").unwrap());
+            &CrateDep::External(VersionReq::parse("1.0").unwrap()));
+        assert_eq!(output.crates["futures-cpupool"].main.get("futures").unwrap(),
+            &CrateDep::Internal(RelativePath::new("..").to_relative_path_buf()));
         assert_eq!(output.crates["futures-cpupool"].dev.len(), 0);
         assert_eq!(output.crates["futures-cpupool"].build.len(), 0);
     }

src/engine/mod.rs

@@ -1,5 +1,4 @@
 use std::collections::HashSet;
-use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -9,6 +8,7 @@ use futures::future::join_all;
 use hyper::Client;
 use hyper::client::HttpConnector;
 use hyper_tls::HttpsConnector;
+use relative_path::{RelativePath, RelativePathBuf};
 use slog::Logger;
 use tokio_service::Service;
 
@@ -70,7 +70,7 @@ impl Engine {
     pub fn analyze_dependencies(&self, repo_path: RepoPath) ->
         impl Future<Item=AnalyzeDependenciesOutcome, Error=Error>
     {
-        let entry_point = PathBuf::from("/");
+        let entry_point = RelativePath::new("/").to_relative_path_buf();
         let manifest_future = CrawlManifestFuture::new(self, repo_path, entry_point);
 
         let engine = self.clone();
@@ -95,10 +95,11 @@ impl Engine {
         })
     }
 
-    fn retrieve_manifest_at_path<P: AsRef<Path>>(&self, repo_path: &RepoPath, path: &P) ->
+    fn retrieve_manifest_at_path(&self, repo_path: &RepoPath, path: &RelativePathBuf) ->
         impl Future<Item=String, Error=Error>
     {
-        retrieve_file_at_path(self.client.clone(), &repo_path, &path.as_ref().join("Cargo.toml")).from_err()
+        let manifest_path = path.join(RelativePath::new("Cargo.toml"));
+        retrieve_file_at_path(self.client.clone(), &repo_path, &manifest_path).from_err()
     }
 }
 

src/interactors/crates.rs

@@ -1,6 +1,6 @@
 use failure::Error;
 use futures::{Future, Stream, IntoFuture, future};
-use hyper::{Error as HyperError, Method, Request, Response};
+use hyper::{Error as HyperError, Method, Request, Response, Uri};
 use tokio_service::Service;
 use semver::Version;
 use serde_json;
@@ -43,15 +43,15 @@ pub fn query_crate<S>(service: S, crate_name: CrateName) ->
     where S: Service<Request=Request, Response=Response, Error=HyperError>
 {
     let uri_future = format!("{}/crates/{}/versions", CRATES_API_BASE_URI, crate_name.as_ref())
-        .parse().into_future().from_err();
+        .parse::<Uri>().into_future().from_err();
 
     uri_future.and_then(move |uri| {
-        let request = Request::new(Method::Get, uri);
+        let request = Request::new(Method::Get, uri.clone());
 
         service.call(request).from_err().and_then(move |response| {
             let status = response.status();
             if !status.is_success() {
-                future::Either::A(future::err(format_err!("Status code: {}", status)))
+                future::Either::A(future::err(format_err!("Status code {} for URI {}", status, uri)))
             } else {
                 let body_future = response.body().concat2().from_err();
                 let decode_future = body_future.and_then(|body| {

src/interactors/github.rs

@@ -1,9 +1,8 @@
-use std::path::Path;
-
 use failure::Error;
 use futures::{Future, IntoFuture, Stream, future};
 use hyper::{Error as HyperError, Method, Request, Response, Uri};
 use hyper::header::UserAgent;
+use relative_path::RelativePathBuf;
 use tokio_service::Service;
 use serde_json;
 
@@ -12,11 +11,11 @@ use ::models::repo::{Repository, RepoPath};
 const GITHUB_API_BASE_URI: &'static str = "https://api.github.com";
 const GITHUB_USER_CONTENT_BASE_URI: &'static str = "https://raw.githubusercontent.com";
 
-pub fn retrieve_file_at_path<S, P: AsRef<Path>>(service: S, repo_path: &RepoPath, path: &P) ->
+pub fn retrieve_file_at_path<S>(service: S, repo_path: &RepoPath, path: &RelativePathBuf) ->
     impl Future<Item=String, Error=Error>
     where S: Service<Request=Request, Response=Response, Error=HyperError>
 {
-    let path_str = path.as_ref().to_str().expect("failed to convert path to str");
+    let path_str: &str = path.as_ref();
     let uri_future = format!("{}/{}/{}/master/{}",
         GITHUB_USER_CONTENT_BASE_URI,
         repo_path.qual.as_ref(),
@@ -83,7 +82,7 @@ impl<S> Service for GetPopularRepos<S>
             service.call(request).from_err().and_then(|response| {
                 let status = response.status();
                 if !status.is_success() {
-                    future::Either::A(future::err(format_err!("Status code: {}", status)))
+                    future::Either::A(future::err(format_err!("Status code {} for popular repo search", status)))
                 } else {
                     let body_future = response.body().concat2().from_err();
                     let decode_future = body_future

src/main.rs

@@ -11,6 +11,7 @@ extern crate hyper_tls;
 #[macro_use] extern crate lazy_static;
 extern crate maud;
 extern crate ordermap;
+extern crate relative_path;
 extern crate route_recognizer;
 extern crate semver;
 #[macro_use] extern crate serde_derive;

src/models/crates.rs

@@ -1,9 +1,9 @@
 use std::borrow::Borrow;
 use std::collections::BTreeMap;
-use std::path::PathBuf;
 use std::str::FromStr;
 
 use failure::Error;
+use relative_path::RelativePathBuf;
 use semver::{Version, VersionReq};
 
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -50,11 +50,27 @@ pub struct CrateRelease {
     pub yanked: bool
 }
 
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum CrateDep {
+    External(VersionReq),
+    Internal(RelativePathBuf)
+}
+
+impl CrateDep {
+    pub fn is_external(&self) -> bool {
+        if let &CrateDep::External(_) = self {
+            true
+        } else {
+            false
+        }
+    }
+}
+
 #[derive(Clone, Debug, Default)]
 pub struct CrateDeps {
-    pub main: BTreeMap<CrateName, VersionReq>,
+    pub main: BTreeMap<CrateName, CrateDep>,
-    pub dev: BTreeMap<CrateName, VersionReq>,
+    pub dev: BTreeMap<CrateName, CrateDep>,
-    pub build: BTreeMap<CrateName, VersionReq>
+    pub build: BTreeMap<CrateName, CrateDep>
 }
 
 #[derive(Debug)]
@@ -87,14 +103,26 @@ pub struct AnalyzedDependencies {
 
 impl AnalyzedDependencies {
     pub fn new(deps: &CrateDeps) -> AnalyzedDependencies {
-        let main = deps.main.iter().map(|(name, req)| {
+        let main = deps.main.iter().filter_map(|(name, dep)| {
-            (name.clone(), AnalyzedDependency::new(req.clone()))
+            if let &CrateDep::External(ref req) = dep {
+                Some((name.clone(), AnalyzedDependency::new(req.clone())))
+            } else {
+                None
+            }
         }).collect();
-        let dev = deps.dev.iter().map(|(name, req)| {
+        let dev = deps.dev.iter().filter_map(|(name, dep)| {
-            (name.clone(), AnalyzedDependency::new(req.clone()))
+            if let &CrateDep::External(ref req) = dep {
+                Some((name.clone(), AnalyzedDependency::new(req.clone())))
+            } else {
+                None
+            }
         }).collect();
-        let build = deps.build.iter().map(|(name, req)| {
+        let build = deps.build.iter().filter_map(|(name, dep)| {
-            (name.clone(), AnalyzedDependency::new(req.clone()))
+            if let &CrateDep::External(ref req) = dep {
+                Some((name.clone(), AnalyzedDependency::new(req.clone())))
+            } else {
+                None
+            }
         }).collect();
         AnalyzedDependencies { main, dev, build }
     }
@@ -113,6 +141,6 @@ impl AnalyzedDependencies {
 #[derive(Clone, Debug)]
 pub enum CrateManifest {
     Package(CrateName, CrateDeps),
-    Workspace { members: Vec<PathBuf> },
+    Workspace { members: Vec<RelativePathBuf> },
-    Mixed { name: CrateName, deps: CrateDeps, members: Vec<PathBuf> }
+    Mixed { name: CrateName, deps: CrateDeps, members: Vec<RelativePathBuf> }
 }

src/parsers/manifest.rs

@@ -1,16 +1,16 @@
 use std::collections::BTreeMap;
-use std::path::PathBuf;
 
 use failure::Error;
+use relative_path::RelativePathBuf;
 use semver::VersionReq;
 use toml;
 
-use ::models::crates::{CrateName, CrateDeps, CrateManifest};
+use ::models::crates::{CrateName, CrateDep, CrateDeps, CrateManifest};
 
 #[derive(Serialize, Deserialize, Debug)]
 struct CargoTomlComplexDependency {
     git: Option<String>,
-    path: Option<String>,
+    path: Option<RelativePathBuf>,
     version: Option<String>
 }
 
@@ -28,7 +28,7 @@ struct CargoTomlPackage {
 
 #[derive(Serialize, Deserialize, Debug)]
 struct CargoTomlWorkspace {
-    members: Vec<PathBuf>
+    members: Vec<RelativePathBuf>
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -47,22 +47,28 @@ struct CargoToml {
     build_dependencies: BTreeMap<String, CargoTomlDependency>
 }
 
-fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option<Result<(CrateName, VersionReq), Error>> {
+fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option<Result<(CrateName, CrateDep), Error>> {
     match cargo_dep {
         (name, CargoTomlDependency::Simple(string)) => {
             Some(name.parse::<CrateName>().map_err(|err| err.into()).and_then(|parsed_name| {
                 string.parse::<VersionReq>().map_err(|err| err.into())
-                    .map(|version| (parsed_name, version))
+                    .map(|version| (parsed_name, CrateDep::External(version)))
             }))
         }
         (name, CargoTomlDependency::Complex(cplx)) => {
-            if cplx.git.is_some() || cplx.path.is_some() {
+            if cplx.git.is_some() {
                 None
+            } else if cplx.path.is_some() {
+                cplx.path.map(|path| {
+                    name.parse::<CrateName>().map_err(|err| err.into()).map(|parsed_name| {
+                        (parsed_name, CrateDep::Internal(path))
+                    })
+                })
             } else {
                 cplx.version.map(|string| {
                     name.parse::<CrateName>().map_err(|err| err.into()).and_then(|parsed_name| {
                         string.parse::<VersionReq>().map_err(|err| err.into())
-                            .map(|version| (parsed_name, version))
+                            .map(|version| (parsed_name, CrateDep::External(version)))
                     })
                 })
             }

src/server/views/html/status.rs

@@ -17,7 +17,7 @@ fn dependency_tables(crate_name: CrateName, deps: AnalyzedDependencies) -> Marku
         }
 
         @if deps.main.is_empty() && deps.dev.is_empty() && deps.build.is_empty() {
-            p class="notification has-text-centered" "No dependencies! 🙌"
+            p class="notification has-text-centered" "No external dependencies! 🙌"
         }
 
         @if !deps.main.is_empty() {