feliix42/deps.rs
1
0
Fork 0
mirror of https://github.com/deps-rs/deps.rs.git synced 2024-11-25 03:26:30 +00:00
Code Issues Projects Releases Packages Wiki Activity

move to the failure crate for error reporting

Browse source
This commit is contained in:
Sam Rijs 2018-02-05 20:23:40 +11:00
parent b101a3fade
commit d935ca0ef8
10 changed files with 100 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -5,6 +5,7 @@ authors = ["Sam Rijs <srijs@airpost.net>"]
[dependencies]
base64 = "0.9.0"
failure = "0.1.1"
futures = "0.1.18"
hyper = "0.11.15"
hyper-tls = "0.1.2"

41
src/engine/mod.rs
View file

@ -1,6 +1,7 @@
use std::sync::Arc;
use std::time::Duration;
use failure::Error;
use futures::{Future, Stream, stream};
use hyper::Client;
use hyper::client::HttpConnector;
@ -10,15 +11,15 @@ use tokio_service::Service;
mod analyzer;
use ::utils::throttle::{Throttle, ThrottledError};
use ::utils::throttle::Throttle;
use ::models::repo::{Repository, RepoPath};
use ::models::crates::{CrateName, CrateRelease, CrateManifest, AnalyzedDependencies};
use ::parsers::manifest::{ManifestParseError, parse_manifest_toml};
use ::parsers::manifest::parse_manifest_toml;
use ::interactors::crates::{QueryCrateError, query_crate};
use ::interactors::github::{RetrieveFileAtPathError, retrieve_file_at_path};
use ::interactors::crates::query_crate;
use ::interactors::github::retrieve_file_at_path;
use ::interactors::github::GetPopularRepos;
use self::analyzer::DependencyAnalyzer;
@ -41,16 +42,6 @@ impl Engine {
}
}
#[derive(Debug)]
pub struct GetPopularReposError(ThrottledError<::interactors::github::GetPopularReposError>);
#[derive(Debug)]
pub enum AnalyzeDependenciesError {
QueryCrate(QueryCrateError),
RetrieveFileAtPath(RetrieveFileAtPathError),
ParseManifest(ManifestParseError)
}
const FETCH_RELEASES_CONCURRENCY: usize = 10;
pub struct AnalyzeDependenciesOutcome {
@ -60,15 +51,14 @@ pub struct AnalyzeDependenciesOutcome {
impl Engine {
pub fn get_popular_repos(&self) ->
impl Future<Item=Vec<Repository>, Error=GetPopularReposError>
impl Future<Item=Vec<Repository>, Error=Error>
{
self.get_popular_repos.call(())
.map_err(GetPopularReposError)
.map(|repos| repos.clone())
.from_err().map(|repos| repos.clone())
}
pub fn analyze_dependencies(&self, repo_path: RepoPath) ->
impl Future<Item=AnalyzeDependenciesOutcome, Error=AnalyzeDependenciesError>
impl Future<Item=AnalyzeDependenciesOutcome, Error=Error>
{
let manifest_future = self.retrieve_manifest(&repo_path);
@ -83,9 +73,9 @@ impl Engine {
let release_futures = engine.fetch_releases(main_deps.chain(dev_deps).chain(build_deps));
let analyzed_deps_future = stream::iter_ok(release_futures)
let analyzed_deps_future = stream::iter_ok::<_, Error>(release_futures)
.buffer_unordered(FETCH_RELEASES_CONCURRENCY)
.fold(analyzer, |mut analyzer, releases| { analyzer.process(releases); Ok(analyzer) })
.fold(analyzer, |mut analyzer, releases| { analyzer.process(releases); Ok(analyzer) as Result<_, Error> })
.map(|analyzer| analyzer.finalize());
analyzed_deps_future.map(move |analyzed_deps| {
@ -98,24 +88,21 @@ impl Engine {
}
fn fetch_releases<I: IntoIterator<Item=CrateName>>(&self, names: I) ->
impl Iterator<Item=impl Future<Item=Vec<CrateRelease>, Error=AnalyzeDependenciesError>>
impl Iterator<Item=impl Future<Item=Vec<CrateRelease>, Error=Error>>
{
let client = self.client.clone();
names.into_iter().map(move |name| {
query_crate(client.clone(), name)
.map_err(AnalyzeDependenciesError::QueryCrate)
.map(|resp| resp.releases)
})
}
fn retrieve_manifest(&self, repo_path: &RepoPath) ->
impl Future<Item=CrateManifest, Error=AnalyzeDependenciesError>
impl Future<Item=CrateManifest, Error=Error>
{
retrieve_file_at_path(self.client.clone(), &repo_path, "Cargo.toml")
.map_err(AnalyzeDependenciesError::RetrieveFileAtPath)
retrieve_file_at_path(self.client.clone(), &repo_path, "Cargo.toml").from_err()
.and_then(|manifest_source| {
parse_manifest_toml(&manifest_source)
.map_err(AnalyzeDependenciesError::ParseManifest)
parse_manifest_toml(&manifest_source).map_err(|err| err.into())
})
}
}

26
src/interactors/crates.rs
View file

@ -1,6 +1,6 @@
use failure::Error;
use futures::{Future, Stream, IntoFuture, future};
use hyper::{Error as HyperError, Method, Request, Response, StatusCode};
use hyper::error::UriError;
use hyper::{Error as HyperError, Method, Request, Response};
use tokio_service::Service;
use semver::Version;
use serde_json;
@ -20,7 +20,7 @@ struct QueryCratesVersionsBody {
versions: Vec<CratesVersion>
}
fn convert_body(name: &CrateName, body: QueryCratesVersionsBody) -> Result<QueryCrateResponse, QueryCrateError> {
fn convert_body(name: &CrateName, body: QueryCratesVersionsBody) -> Result<QueryCrateResponse, Error> {
let releases = body.versions.into_iter().map(|version| {
CrateRelease {
name: name.clone(),
@ -38,33 +38,25 @@ pub struct QueryCrateResponse {
pub releases: Vec<CrateRelease>
}
#[derive(Debug)]
pub enum QueryCrateError {
Uri(UriError),
Status(StatusCode),
Transport(HyperError),
Decode(serde_json::Error)
}
pub fn query_crate<S>(service: S, crate_name: CrateName) ->
impl Future<Item=QueryCrateResponse, Error=QueryCrateError>
impl Future<Item=QueryCrateResponse, Error=Error>
where S: Service<Request=Request, Response=Response, Error=HyperError>
{
let uri_future = format!("{}/crates/{}/versions", CRATES_API_BASE_URI, crate_name.as_ref())
.parse().into_future().map_err(QueryCrateError::Uri);
.parse().into_future().from_err();
uri_future.and_then(move |uri| {
let request = Request::new(Method::Get, uri);
service.call(request).map_err(QueryCrateError::Transport).and_then(move |response| {
service.call(request).from_err().and_then(move |response| {
let status = response.status();
if !status.is_success() {
future::Either::A(future::err(QueryCrateError::Status(status)))
future::Either::A(future::err(format_err!("Status code: {}", status)))
} else {
let body_future = response.body().concat2().map_err(QueryCrateError::Transport);
let body_future = response.body().concat2().from_err();
let decode_future = body_future.and_then(|body| {
serde_json::from_slice::<QueryCratesVersionsBody>(&body)
.map_err(QueryCrateError::Decode)
.map_err(|err| err.into())
});
let convert_future = decode_future.and_then(move |body| convert_body(&crate_name, body));
future::Either::B(convert_future)

52
src/interactors/github.rs
View file

@ -1,27 +1,17 @@
use std::string::FromUtf8Error;
use failure::Error;
use futures::{Future, IntoFuture, Stream, future};
use hyper::{Error as HyperError, Method, Request, Response, StatusCode};
use hyper::error::UriError;
use hyper::{Error as HyperError, Method, Request, Response};
use hyper::header::UserAgent;
use tokio_service::Service;
use serde_json;
use ::models::repo::{Repository, RepoPath, RepoValidationError};
use ::models::repo::{Repository, RepoPath};
const GITHUB_API_BASE_URI: &'static str = "https://api.github.com";
const GITHUB_USER_CONTENT_BASE_URI: &'static str = "https://raw.githubusercontent.com";
#[derive(Debug)]
pub enum RetrieveFileAtPathError {
Uri(UriError),
Transport(HyperError),
Status(StatusCode),
Decode(FromUtf8Error)
}
pub fn retrieve_file_at_path<S>(service: S, repo_path: &RepoPath, file_path: &str) ->
impl Future<Item=String, Error=RetrieveFileAtPathError>
impl Future<Item=String, Error=Error>
where S: Service<Request=Request, Response=Response, Error=HyperError>
{
let uri_future = format!("{}/{}/{}/master/{}",
@ -29,34 +19,25 @@ pub fn retrieve_file_at_path<S>(service: S, repo_path: &RepoPath, file_path: &st
repo_path.qual.as_ref(),
repo_path.name.as_ref(),
file_path
).parse().into_future().map_err(RetrieveFileAtPathError::Uri);
).parse().into_future().from_err();
uri_future.and_then(move |uri| {
let request = Request::new(Method::Get, uri);
service.call(request).map_err(RetrieveFileAtPathError::Transport).and_then(|response| {
service.call(request).from_err().and_then(|response| {
let status = response.status();
if !status.is_success() {
future::Either::A(future::err(RetrieveFileAtPathError::Status(status)))
future::Either::A(future::err(format_err!("Status code: {}", status)))
} else {
let body_future = response.body().concat2().map_err(RetrieveFileAtPathError::Transport);
let body_future = response.body().concat2().from_err();
let decode_future = body_future
.and_then(|body| String::from_utf8(body.to_vec()).map_err(RetrieveFileAtPathError::Decode));
.and_then(|body| String::from_utf8(body.to_vec()).map_err(|err| err.into()));
future::Either::B(decode_future)
}
})
})
}
#[derive(Debug)]
pub enum GetPopularReposError {
Uri(UriError),
Transport(HyperError),
Status(StatusCode),
Decode(serde_json::Error),
Validate(RepoValidationError)
}
#[derive(Deserialize)]
struct GithubSearchResponse {
items: Vec<GithubRepo>
@ -83,7 +64,7 @@ impl<S> Service for GetPopularRepos<S>
{
type Request = ();
type Response = Vec<Repository>;
type Error = GetPopularReposError;
type Error = Error;
type Future = Box<Future<Item=Self::Response, Error=Self::Error>>;
fn call(&self, _req: ()) -> Self::Future {
@ -91,24 +72,23 @@ impl<S> Service for GetPopularRepos<S>
let service = self.0.clone();
let uri_future = format!("{}/search/repositories?q=language:rust&sort=stars", GITHUB_API_BASE_URI)
.parse().into_future().map_err(GetPopularReposError::Uri);
.parse().into_future().from_err();
Box::new(uri_future.and_then(move |uri| {
let mut request = Request::new(Method::Get, uri);
request.headers_mut().set(UserAgent::new("deps.rs"));
service.call(request).map_err(GetPopularReposError::Transport).and_then(|response| {
service.call(request).from_err().and_then(|response| {
let status = response.status();
if !status.is_success() {
future::Either::A(future::err(GetPopularReposError::Status(status)))
future::Either::A(future::err(format_err!("Status code: {}", status)))
} else {
let body_future = response.body().concat2().map_err(GetPopularReposError::Transport);
let body_future = response.body().concat2().from_err();
let decode_future = body_future
.and_then(|body| serde_json::from_slice(body.as_ref()).map_err(GetPopularReposError::Decode));
.and_then(|body| serde_json::from_slice(body.as_ref()).map_err(|err| err.into()));
future::Either::B(decode_future.and_then(|search_response: GithubSearchResponse| {
search_response.items.into_iter().map(|item| {
let path = RepoPath::from_parts("github", &item.owner.login, &item.name)
.map_err(GetPopularReposError::Validate)?;
let path = RepoPath::from_parts("github", &item.owner.login, &item.name)?;
Ok(Repository { path, description: item.description })
}).collect::<Result<Vec<_>, _>>()
}))

1
src/main.rs
View file

@ -4,6 +4,7 @@
#![feature(proc_macro)]
extern crate base64;
#[macro_use] extern crate failure;
extern crate futures;
extern crate hyper;
extern crate hyper_tls;

10
src/models/crates.rs
View file

@ -2,6 +2,7 @@ use std::borrow::Borrow;
use std::collections::BTreeMap;
use std::str::FromStr;
use failure::Error;
use semver::{Version, VersionReq};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
@ -25,19 +26,16 @@ impl AsRef<str> for CrateName {
}
}
#[derive(Debug)]
pub struct CrateNameValidationError;
impl FromStr for CrateName {
type Err = CrateNameValidationError;
type Err = Error;
fn from_str(input: &str) -> Result<CrateName, CrateNameValidationError> {
fn from_str(input: &str) -> Result<CrateName, Error> {
let is_valid = input.chars().all(|c| {
c.is_ascii_alphanumeric() || c == '_' || c == '-'
});
if !is_valid {
Err(CrateNameValidationError)
Err(format_err!("failed to validate crate name: {}", input))
} else {
Ok(CrateName(input.to_string()))
}

31
src/models/repo.rs
View file

@ -1,5 +1,7 @@
use std::str::FromStr;
use failure::Error;
#[derive(Clone, Debug)]
pub struct Repository {
pub path: RepoPath,
@ -14,7 +16,7 @@ pub struct RepoPath {
}
impl RepoPath {
pub fn from_parts(site: &str, qual: &str, name: &str) -> Result<RepoPath, RepoValidationError> {
pub fn from_parts(site: &str, qual: &str, name: &str) -> Result<RepoPath, Error> {
Ok(RepoPath {
site: site.parse()?,
qual: qual.parse()?,
@ -23,9 +25,6 @@ impl RepoPath {
}
}
#[derive(Debug)]
pub struct RepoValidationError;
#[derive(Clone, Copy, Debug)]
pub enum RepoSite {
Github
@ -40,12 +39,12 @@ impl RepoSite {
}
impl FromStr for RepoSite {
type Err = RepoValidationError;
type Err = Error;
fn from_str(input: &str) -> Result<RepoSite, RepoValidationError> {
fn from_str(input: &str) -> Result<RepoSite, Error> {
match input {
"github" => Ok(RepoSite::Github),
_ => Err(RepoValidationError)
_ => Err(format_err!("unknown repo site identifier"))
}
}
}
@ -62,19 +61,16 @@ impl AsRef<str> for RepoSite {
pub struct RepoQualifier(String);
impl FromStr for RepoQualifier {
type Err = RepoValidationError;
type Err = Error;
fn from_str(input: &str) -> Result<RepoQualifier, RepoValidationError> {
fn from_str(input: &str) -> Result<RepoQualifier, Error> {
let is_valid = input.chars().all(|c| {
c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == '_'
});
if !is_valid {
Err(RepoValidationError)
} else {
ensure!(is_valid, "invalid repo qualifier");
Ok(RepoQualifier(input.to_string()))
}
}
}
impl AsRef<str> for RepoQualifier {
@ -87,19 +83,16 @@ impl AsRef<str> for RepoQualifier {
pub struct RepoName(String);
impl FromStr for RepoName {
type Err = RepoValidationError;
type Err = Error;
fn from_str(input: &str) -> Result<RepoName, RepoValidationError> {
fn from_str(input: &str) -> Result<RepoName, Error> {
let is_valid = input.chars().all(|c| {
c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == '_'
});
if !is_valid {
Err(RepoValidationError)
} else {
ensure!(is_valid, "invalid repo name");
Ok(RepoName(input.to_string()))
}
}
}
impl AsRef<str> for RepoName {

30
src/parsers/manifest.rs
View file

@ -1,16 +1,10 @@
use std::collections::BTreeMap;
use semver::{ReqParseError, VersionReq};
use failure::Error;
use semver::VersionReq;
use toml;
use ::models::crates::{CrateName, CrateDeps, CrateManifest, CrateNameValidationError};
#[derive(Debug)]
pub enum ManifestParseError {
Serde(toml::de::Error),
Name(CrateNameValidationError),
Version(ReqParseError)
}
use ::models::crates::{CrateName, CrateDeps, CrateManifest};
#[derive(Serialize, Deserialize, Debug)]
struct CargoTomlComplexDependency {
@ -44,11 +38,11 @@ struct CargoToml {
build_dependencies: BTreeMap<String, CargoTomlDependency>
}
fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option<Result<(CrateName, VersionReq), ManifestParseError>> {
fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option<Result<(CrateName, VersionReq), Error>> {
match cargo_dep {
(name, CargoTomlDependency::Simple(string)) => {
Some(name.parse().map_err(ManifestParseError::Name).and_then(|parsed_name| {
string.parse().map_err(ManifestParseError::Version)
Some(name.parse::<CrateName>().map_err(|err| err.into()).and_then(|parsed_name| {
string.parse::<VersionReq>().map_err(|err| err.into())
.map(|version| (parsed_name, version))
}))
}
@ -57,8 +51,8 @@ fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option<Result
None
} else {
cplx.version.map(|string| {
name.parse().map_err(ManifestParseError::Name).and_then(|parsed_name| {
string.parse().map_err(ManifestParseError::Version)
name.parse::<CrateName>().map_err(|err| err.into()).and_then(|parsed_name| {
string.parse::<VersionReq>().map_err(|err| err.into())
.map(|version| (parsed_name, version))
})
})
@ -67,12 +61,10 @@ fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option<Result
}
}
pub fn parse_manifest_toml(input: &str) -> Result<CrateManifest, ManifestParseError> {
let cargo_toml = toml::de::from_str::<CargoToml>(input)
.map_err(ManifestParseError::Serde)?;
pub fn parse_manifest_toml(input: &str) -> Result<CrateManifest, Error> {
let cargo_toml = toml::de::from_str::<CargoToml>(input)?;
let crate_name = cargo_toml.package.name.parse()
.map_err(ManifestParseError::Name)?;
let crate_name = cargo_toml.package.name.parse::<CrateName>()?;
let dependencies = cargo_toml.dependencies
.into_iter().filter_map(convert_dependency).collect::<Result<BTreeMap<_, _>, _>>()?;

6
src/server/mod.rs
View file

@ -98,7 +98,7 @@ impl Server {
Err(err) => {
let mut response = Response::new();
response.set_status(StatusCode::BadRequest);
response.set_body(format!("{:?}", err));
response.set_body(format!("Error: {}", err));
future::ok(response)
},
Ok(popular) =>
@ -121,7 +121,7 @@ impl Server {
Err(err) => {
let mut response = Response::new();
response.set_status(StatusCode::BadRequest);
response.set_body(format!("{:?}", err));
response.set_body(format!("Error: {}", err));
future::Either::A(future::ok(response))
},
Ok(repo_path) => {
@ -131,7 +131,7 @@ impl Server {
if format != StatusFormat::Svg {
let mut response = Response::new();
response.set_status(StatusCode::BadRequest);
response.set_body(format!("{:?}", err));
response.set_body(format!("Error: {}", err));
future::Either::A(future::ok(response))
} else {
future::Either::A(future::ok(views::status_svg(None)))

56
src/utils/throttle.rs
View file

@ -1,21 +1,23 @@
use std::error::Error;
use std::fmt::{Debug, Display, Formatter, Result as FmtResult};
use std::time::{Duration, Instant};
use std::ops::Deref;
use std::sync::Mutex;
use failure::{Error, Fail};
use futures::{Future, Poll};
use futures::future::{Shared, SharedError, SharedItem};
use tokio_service::Service;
pub struct Throttle<S: Service<Request=()>> {
pub struct Throttle<S>
where S: Service<Request=(), Error=Error>
{
inner: S,
duration: Duration,
current: Mutex<Option<(Instant, Shared<S::Future>)>>
}
impl<S> Debug for Throttle<S>
where S: Service<Request=()> + Debug,
where S: Service<Request=(), Error=Error> + Debug
{
fn fmt(&self, fmt: &mut Formatter) -> FmtResult {
fmt.debug_struct("Throttle")
@ -25,7 +27,9 @@ impl<S> Debug for Throttle<S>
}
}
impl<S: Service<Request=()>> Throttle<S> {
impl<S> Throttle<S>
where S: Service<Request=(), Error=Error>
{
pub fn new(service: S, duration: Duration) -> Throttle<S> {
Throttle {
inner: service,
@ -35,10 +39,12 @@ impl<S: Service<Request=()>> Throttle<S> {
}
}
impl<S: Service<Request=()>> Service for Throttle<S> {
impl<S> Service for Throttle<S>
where S: Service<Request=(), Error=Error>
{
type Request = ();
type Response = ThrottledItem<S::Response>;
type Error = ThrottledError<S::Error>;
type Error = ThrottledError;
type Future = Throttled<S::Future>;
fn call(&self, _: ()) -> Self::Future {
@ -69,9 +75,9 @@ impl<F> Debug for Throttled<F>
}
}
impl<F: Future> Future for Throttled<F> {
impl<F: Future<Error=Error>> Future for Throttled<F> {
type Item = ThrottledItem<F::Item>;
type Error = ThrottledError<F::Error>;
type Error = ThrottledError;
fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
self.0.poll()
@ -92,32 +98,24 @@ impl<T> Deref for ThrottledItem<T> {
}
#[derive(Debug)]
pub struct ThrottledError<E>(SharedError<E>);
pub struct ThrottledError(SharedError<Error>);
impl<E> Deref for ThrottledError<E> {
type Target = E;
impl Fail for ThrottledError {
fn cause(&self) -> Option<&Fail> {
Some(self.0.cause())
}
fn deref(&self) -> &E {
&self.0.deref()
fn backtrace(&self) -> Option<&::failure::Backtrace> {
Some(self.0.backtrace())
}
fn causes(&self) -> ::failure::Causes {
self.0.causes()
}
}
impl<E> Display for ThrottledError<E>
where E: Display,
{
impl Display for ThrottledError {
fn fmt(&self, f: &mut Formatter) -> FmtResult {
self.0.fmt(f)
}
}
impl<E> Error for ThrottledError<E>
where E: Error,
{
fn description(&self) -> &str {
self.0.description()
}
fn cause(&self) -> Option<&Error> {
self.0.cause()
Display::fmt(&self.0, f)
}
}