first version of api is running

This commit is contained in:
Sam Rijs 2018-01-26 23:15:53 +11:00
parent f82e3d0ef6
commit b6dea8b0e9
16 changed files with 613 additions and 105 deletions

View file

@ -7,6 +7,7 @@ authors = ["Sam Rijs <srijs@airpost.net>"]
futures = "0.1.18"
hyper = "0.11.15"
hyper-tls = "0.1.2"
semver = { version = "0.9.0", features = ["serde"] }
serde = "1.0.27"
serde_derive = "1.0.27"
serde_json = "1.0.9"
@ -14,3 +15,4 @@ slog = "2.1.1"
slog-json = "2.2.0"
tokio-core = "0.1.12"
tokio-service = "0.1.0"
toml = "0.4.5"

84
src/api.rs Normal file
View file

@ -0,0 +1,84 @@
use std::collections::BTreeMap;
use futures::{Future, future};
use hyper::{Client, Error as HyperError, Request, Response, StatusCode};
use hyper::client::HttpConnector;
use hyper::header::ContentType;
use hyper_tls::HttpsConnector;
use semver::{Version, VersionReq};
use serde_json;
use slog::Logger;
use tokio_service::Service;
use ::models::repo::RepoPath;
use ::engine::Engine;
pub struct Api {
pub engine: Engine
}
#[derive(Debug, Serialize)]
struct AnalyzeDependenciesResponseDetail {
required: VersionReq,
latest: Option<Version>,
outdated: bool
}
#[derive(Debug, Serialize)]
struct AnalyzeDependenciesResponse {
dependencies: BTreeMap<String, AnalyzeDependenciesResponseDetail>,
#[serde(rename="dev-dependencies")]
dev_dependencies: BTreeMap<String, AnalyzeDependenciesResponseDetail>,
#[serde(rename="build-dependencies")]
build_dependencies: BTreeMap<String, AnalyzeDependenciesResponseDetail>
}
impl Service for Api {
type Request = Request;
type Response = Response;
type Error = HyperError;
type Future = Box<Future<Item=Response, Error=HyperError>>;
fn call(&self, req: Request) -> Self::Future {
let repo_path = RepoPath::from_parts("github.com", "hyperium", "hyper").unwrap();
let future = self.engine.analyze_dependencies(repo_path).then(|result| {
match result {
Err(err) => {
let mut response = Response::new();
response.set_status(StatusCode::InternalServerError);
response.set_body(format!("{:?}", err));
future::Either::A(future::ok(response))
},
Ok(dependencies) => {
let response_struct = AnalyzeDependenciesResponse {
dependencies: dependencies.main.into_iter()
.map(|(name, analyzed)| (name.into(), AnalyzeDependenciesResponseDetail {
outdated: analyzed.is_outdated(),
required: analyzed.required,
latest: analyzed.latest
})).collect(),
dev_dependencies: dependencies.dev.into_iter()
.map(|(name, analyzed)| (name.into(), AnalyzeDependenciesResponseDetail {
outdated: analyzed.is_outdated(),
required: analyzed.required,
latest: analyzed.latest
})).collect(),
build_dependencies: dependencies.build.into_iter()
.map(|(name, analyzed)| (name.into(), AnalyzeDependenciesResponseDetail {
outdated: analyzed.is_outdated(),
required: analyzed.required,
latest: analyzed.latest
})).collect()
};
let mut response = Response::new()
.with_header(ContentType::json())
.with_body(serde_json::to_string(&response_struct).unwrap());
future::Either::B(future::ok(response))
}
}
});
Box::new(future)
}
}

55
src/engine/analyzer.rs Normal file
View file

@ -0,0 +1,55 @@
use std::collections::BTreeMap;
use std::collections::btree_map::{Entry as BTreeMapEntry};
use semver::{Version, VersionReq};
use ::models::crates::{CrateName, CrateDeps, CrateRelease, AnalyzedDependency, AnalyzedDependencies};
pub struct DependencyAnalyzer {
deps: AnalyzedDependencies
}
impl DependencyAnalyzer {
pub fn new(deps: &CrateDeps) -> DependencyAnalyzer {
DependencyAnalyzer {
deps: AnalyzedDependencies::new(deps)
}
}
fn process_single(dep: &mut AnalyzedDependency, ver: &Version) {
if dep.required.matches(&ver) {
if let Some(ref mut current_latest_that_matches) = dep.latest_that_matches {
if *current_latest_that_matches < *ver {
*current_latest_that_matches = ver.clone();
}
} else {
dep.latest_that_matches = Some(ver.clone());
}
}
if let Some(ref mut current_latest) = dep.latest {
if *current_latest < *ver {
*current_latest = ver.clone();
}
} else {
dep.latest = Some(ver.clone());
}
}
pub fn process<I: IntoIterator<Item=CrateRelease>>(&mut self, releases: I) {
for release in releases.into_iter().filter(|r| !r.yanked) {
if let Some(main_dep) = self.deps.main.get_mut(&release.name) {
DependencyAnalyzer::process_single(main_dep, &release.version)
}
if let Some(dev_dep) = self.deps.dev.get_mut(&release.name) {
DependencyAnalyzer::process_single(dev_dep, &release.version)
}
if let Some(build_dep) = self.deps.build.get_mut(&release.name) {
DependencyAnalyzer::process_single(build_dep, &release.version)
}
}
}
pub fn finalize(self) -> AnalyzedDependencies {
self.deps
}
}

79
src/engine/mod.rs Normal file
View file

@ -0,0 +1,79 @@
mod analyzer;
use futures::{Future, Stream, future, stream};
use hyper::{Client, Error as HyperError, Request, Response, StatusCode};
use hyper::client::HttpConnector;
use hyper_tls::HttpsConnector;
use slog::Logger;
use ::models::repo::RepoPath;
use ::models::crates::{CrateName, CrateRelease, CrateManifest, AnalyzedDependencies};
use ::parsers::manifest::{ManifestParseError, parse_manifest_toml};
use ::interactors::crates::{QueryCrateError, query_crate};
use ::interactors::github::{RetrieveFileAtPathError, retrieve_file_at_path};
use self::analyzer::DependencyAnalyzer;
#[derive(Clone, Debug)]
pub struct Engine {
pub client: Client<HttpsConnector<HttpConnector>>,
pub logger: Logger
}
#[derive(Debug)]
pub enum AnalyzeDependenciesError {
QueryCrate(QueryCrateError),
RetrieveFileAtPath(RetrieveFileAtPathError),
ParseManifest(ManifestParseError)
}
const FETCH_RELEASES_CONCURRENCY: usize = 10;
impl Engine {
pub fn analyze_dependencies(&self, repo_path: RepoPath) ->
impl Future<Item=AnalyzedDependencies, Error=AnalyzeDependenciesError>
{
let manifest_future = self.retrieve_manifest(&repo_path);
let engine = self.clone();
manifest_future.and_then(move |manifest| {
let CrateManifest::Crate(deps) = manifest;
let analyzer = DependencyAnalyzer::new(&deps);
let main_deps = deps.main.into_iter().map(|(name, _)| name);
let dev_deps = deps.dev.into_iter().map(|(name, _)| name);
let build_deps = deps.build.into_iter().map(|(name, _)| name);
let release_futures = engine.fetch_releases(main_deps.chain(dev_deps).chain(build_deps));
stream::iter_ok(release_futures)
.buffer_unordered(FETCH_RELEASES_CONCURRENCY)
.fold(analyzer, |mut analyzer, releases| { analyzer.process(releases); Ok(analyzer) })
.map(|analyzer| analyzer.finalize())
})
}
fn fetch_releases<I: IntoIterator<Item=CrateName>>(&self, names: I) ->
impl Iterator<Item=impl Future<Item=Vec<CrateRelease>, Error=AnalyzeDependenciesError>>
{
let client = self.client.clone();
names.into_iter().map(move |name| {
query_crate(client.clone(), name)
.map_err(AnalyzeDependenciesError::QueryCrate)
.map(|resp| resp.releases)
})
}
fn retrieve_manifest(&self, repo_path: &RepoPath) ->
impl Future<Item=CrateManifest, Error=AnalyzeDependenciesError>
{
retrieve_file_at_path(self.client.clone(), &repo_path, "Cargo.toml")
.map_err(AnalyzeDependenciesError::RetrieveFileAtPath)
.and_then(|manifest_source| {
parse_manifest_toml(&manifest_source)
.map_err(AnalyzeDependenciesError::ParseManifest)
})
}
}

74
src/interactors/crates.rs Normal file
View file

@ -0,0 +1,74 @@
use futures::{Future, Stream, IntoFuture, future};
use hyper::{Error as HyperError, Method, Request, Response, StatusCode};
use hyper::error::UriError;
use tokio_service::Service;
use semver::Version;
use serde_json;
use ::models::crates::{CrateName, CrateRelease};
const CRATES_API_BASE_URI: &'static str = "https://crates.io/api/v1";
#[derive(Serialize, Deserialize, Debug)]
struct CratesVersion {
num: Version,
yanked: bool
}
#[derive(Serialize, Deserialize, Debug)]
struct QueryCratesVersionsBody {
versions: Vec<CratesVersion>
}
fn convert_body(name: &CrateName, body: QueryCratesVersionsBody) -> Result<QueryCrateResponse, QueryCrateError> {
let releases = body.versions.into_iter().map(|version| {
CrateRelease {
name: name.clone(),
version: version.num,
yanked: version.yanked
}
}).collect();
Ok(QueryCrateResponse {
releases: releases
})
}
pub struct QueryCrateResponse {
pub releases: Vec<CrateRelease>
}
#[derive(Debug)]
pub enum QueryCrateError {
Uri(UriError),
Status(StatusCode),
Transport(HyperError),
Decode(serde_json::Error)
}
pub fn query_crate<S>(service: S, crate_name: CrateName) ->
impl Future<Item=QueryCrateResponse, Error=QueryCrateError>
where S: Service<Request=Request, Response=Response, Error=HyperError>
{
let uri_future = format!("{}/crates/{}/versions", CRATES_API_BASE_URI, crate_name.as_ref())
.parse().into_future().map_err(QueryCrateError::Uri);
uri_future.and_then(move |uri| {
let request = Request::new(Method::Get, uri);
service.call(request).map_err(QueryCrateError::Transport).and_then(move |response| {
let status = response.status();
if !status.is_success() {
future::Either::A(future::err(QueryCrateError::Status(status)))
} else {
let body_future = response.body().concat2().map_err(QueryCrateError::Transport);
let decode_future = body_future.and_then(|body| {
serde_json::from_slice::<QueryCratesVersionsBody>(&body)
.map_err(QueryCrateError::Decode)
});
let convert_future = decode_future.and_then(move |body| convert_body(&crate_name, body));
future::Either::B(convert_future)
}
})
})
}

46
src/interactors/github.rs Normal file
View file

@ -0,0 +1,46 @@
use std::string::FromUtf8Error;
use futures::{Future, IntoFuture, Stream, future};
use hyper::{Error as HyperError, Method, Request, Response, StatusCode};
use hyper::error::UriError;
use tokio_service::Service;
use ::models::repo::RepoPath;
const GITHUB_USER_CONTENT_BASE_URI: &'static str = "https://raw.githubusercontent.com";
#[derive(Debug)]
pub enum RetrieveFileAtPathError {
Uri(UriError),
Transport(HyperError),
Status(StatusCode),
Decode(FromUtf8Error)
}
pub fn retrieve_file_at_path<S>(service: S, repo_path: &RepoPath, file_path: &str) ->
impl Future<Item=String, Error=RetrieveFileAtPathError>
where S: Service<Request=Request, Response=Response, Error=HyperError>
{
let uri_future = format!("{}/{}/{}/master/{}",
GITHUB_USER_CONTENT_BASE_URI,
repo_path.qual.as_ref(),
repo_path.name.as_ref(),
file_path
).parse().into_future().map_err(RetrieveFileAtPathError::Uri);
uri_future.and_then(move |uri| {
let request = Request::new(Method::Get, uri);
service.call(request).map_err(RetrieveFileAtPathError::Transport).and_then(|response| {
let status = response.status();
if !status.is_success() {
future::Either::A(future::err(RetrieveFileAtPathError::Status(status)))
} else {
let body_future = response.body().concat2().map_err(RetrieveFileAtPathError::Transport);
let decode_future = body_future
.and_then(|body| String::from_utf8(body.to_vec()).map_err(RetrieveFileAtPathError::Decode));
future::Either::B(decode_future)
}
})
})
}

View file

@ -1 +1,2 @@
pub mod crates;
pub mod github;

View file

@ -4,6 +4,7 @@
extern crate futures;
extern crate hyper;
extern crate hyper_tls;
extern crate semver;
#[macro_use] extern crate serde_derive;
extern crate serde;
extern crate serde_json;
@ -11,10 +12,13 @@ extern crate serde_json;
extern crate slog_json;
extern crate tokio_core;
extern crate tokio_service;
extern crate toml;
mod models;
mod robots;
mod serve;
mod parsers;
mod interactors;
mod engine;
mod api;
use std::net::SocketAddr;
use std::sync::Mutex;
@ -26,7 +30,8 @@ use hyper_tls::HttpsConnector;
use slog::Drain;
use tokio_core::reactor::Core;
use self::serve::Serve;
use self::api::Api;
use self::engine::Engine;
fn main() {
let logger = slog::Logger::root(
@ -51,12 +56,13 @@ fn main() {
let http = Http::new();
let serve_logger = logger.clone();
let engine = Engine {
client: client.clone(),
logger: logger.clone()
};
let serve = http.serve_addr_handle(&addr, &handle, move || {
Ok(Serve {
client: client.clone(),
logger: serve_logger.clone()
})
Ok(Api { engine: engine.clone() })
}).expect("failed to bind server");
let serving = serve.for_each(move |conn| {

View file

@ -1,14 +1,24 @@
use std::collections::BTreeMap;
use std::str::FromStr;
use semver::{Version, VersionReq};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct CrateName(String);
impl Into<String> for CrateName {
fn into(self) -> String {
self.0
}
}
#[derive(Debug)]
pub struct CrateNameValidationError;
impl AsRef<str> for CrateName {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl FromStr for CrateName {
@ -26,3 +36,72 @@ impl FromStr for CrateName {
}
}
}
#[derive(Debug)]
pub struct CrateRelease {
pub name: CrateName,
pub version: Version,
pub yanked: bool
}
#[derive(Clone, Copy, Debug)]
pub enum DependencyType {
Main,
Dev,
Build
}
#[derive(Debug)]
pub struct CrateDeps {
pub main: BTreeMap<CrateName, VersionReq>,
pub dev: BTreeMap<CrateName, VersionReq>,
pub build: BTreeMap<CrateName, VersionReq>
}
#[derive(Debug)]
pub struct AnalyzedDependency {
pub required: VersionReq,
pub latest_that_matches: Option<Version>,
pub latest: Option<Version>
}
impl AnalyzedDependency {
pub fn new(required: VersionReq) -> AnalyzedDependency {
AnalyzedDependency {
required,
latest_that_matches: None,
latest: None
}
}
pub fn is_outdated(&self) -> bool {
self.latest > self.latest_that_matches
}
}
#[derive(Debug)]
pub struct AnalyzedDependencies {
pub main: BTreeMap<CrateName, AnalyzedDependency>,
pub dev: BTreeMap<CrateName, AnalyzedDependency>,
pub build: BTreeMap<CrateName, AnalyzedDependency>
}
impl AnalyzedDependencies {
pub fn new(deps: &CrateDeps) -> AnalyzedDependencies {
let main = deps.main.iter().map(|(name, req)| {
(name.clone(), AnalyzedDependency::new(req.clone()))
}).collect();
let dev = deps.dev.iter().map(|(name, req)| {
(name.clone(), AnalyzedDependency::new(req.clone()))
}).collect();
let build = deps.build.iter().map(|(name, req)| {
(name.clone(), AnalyzedDependency::new(req.clone()))
}).collect();
AnalyzedDependencies { main, dev, build }
}
}
#[derive(Debug)]
pub enum CrateManifest {
Crate(CrateDeps)
}

View file

@ -1 +1,2 @@
pub mod crates;
pub mod repo;

92
src/models/repo.rs Normal file
View file

@ -0,0 +1,92 @@
use std::str::FromStr;
pub struct RepoPath {
pub site: RepoSite,
pub qual: RepoQualifier,
pub name: RepoName
}
impl RepoPath {
pub fn from_parts(site: &str, qual: &str, name: &str) -> Result<RepoPath, RepoValidationError> {
Ok(RepoPath {
site: site.parse()?,
qual: qual.parse()?,
name: name.parse()?
})
}
}
#[derive(Debug)]
pub struct RepoValidationError;
pub struct RepoSite(String);
impl FromStr for RepoSite {
type Err = RepoValidationError;
fn from_str(input: &str) -> Result<RepoSite, RepoValidationError> {
let is_valid = input.chars().all(|c| {
c.is_ascii_alphanumeric() || c == '.'
});
if !is_valid {
Err(RepoValidationError)
} else {
Ok(RepoSite(input.to_string()))
}
}
}
impl AsRef<str> for RepoSite {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
pub struct RepoQualifier(String);
impl FromStr for RepoQualifier {
type Err = RepoValidationError;
fn from_str(input: &str) -> Result<RepoQualifier, RepoValidationError> {
let is_valid = input.chars().all(|c| {
c.is_ascii_alphanumeric() || c == '-' || c == '_'
});
if !is_valid {
Err(RepoValidationError)
} else {
Ok(RepoQualifier(input.to_string()))
}
}
}
impl AsRef<str> for RepoQualifier {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
pub struct RepoName(String);
impl FromStr for RepoName {
type Err = RepoValidationError;
fn from_str(input: &str) -> Result<RepoName, RepoValidationError> {
let is_valid = input.chars().all(|c| {
c.is_ascii_alphanumeric() || c == '-' || c == '_'
});
if !is_valid {
Err(RepoValidationError)
} else {
Ok(RepoName(input.to_string()))
}
}
}
impl AsRef<str> for RepoName {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}

82
src/parsers/manifest.rs Normal file
View file

@ -0,0 +1,82 @@
use std::collections::BTreeMap;
use semver::{ReqParseError, VersionReq};
use toml;
use ::models::crates::{CrateName, CrateDeps, CrateManifest, CrateNameValidationError};
#[derive(Debug)]
pub enum ManifestParseError {
Serde(toml::de::Error),
Name(CrateNameValidationError),
Version(ReqParseError)
}
#[derive(Serialize, Deserialize, Debug)]
struct CargoTomlComplexDependency {
git: Option<String>,
path: Option<String>,
version: Option<String>
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(untagged)]
enum CargoTomlDependency {
Simple(String),
Complex(CargoTomlComplexDependency)
}
#[derive(Serialize, Deserialize, Debug)]
struct CargoToml {
#[serde(default)]
dependencies: BTreeMap<String, CargoTomlDependency>,
#[serde(rename = "dev-dependencies")]
#[serde(default)]
dev_dependencies: BTreeMap<String, CargoTomlDependency>,
#[serde(rename = "build-dependencies")]
#[serde(default)]
build_dependencies: BTreeMap<String, CargoTomlDependency>
}
fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option<Result<(CrateName, VersionReq), ManifestParseError>> {
match cargo_dep {
(name, CargoTomlDependency::Simple(string)) => {
Some(name.parse().map_err(ManifestParseError::Name).and_then(|parsed_name| {
string.parse().map_err(ManifestParseError::Version)
.map(|version| (parsed_name, version))
}))
}
(name, CargoTomlDependency::Complex(cplx)) => {
if cplx.git.is_some() || cplx.path.is_some() {
None
} else {
cplx.version.map(|string| {
name.parse().map_err(ManifestParseError::Name).and_then(|parsed_name| {
string.parse().map_err(ManifestParseError::Version)
.map(|version| (parsed_name, version))
})
})
}
}
}
}
pub fn parse_manifest_toml(input: &str) -> Result<CrateManifest, ManifestParseError> {
let cargo_toml = toml::de::from_str::<CargoToml>(input)
.map_err(ManifestParseError::Serde)?;
let dependencies = cargo_toml.dependencies
.into_iter().filter_map(convert_dependency).collect::<Result<BTreeMap<_, _>, _>>()?;
let dev_dependencies = cargo_toml.dev_dependencies
.into_iter().filter_map(convert_dependency).collect::<Result<BTreeMap<_, _>, _>>()?;
let build_dependencies = cargo_toml.build_dependencies
.into_iter().filter_map(convert_dependency).collect::<Result<BTreeMap<_, _>, _>>()?;
let deps = CrateDeps {
main: dependencies,
dev: dev_dependencies,
build: build_dependencies
};
Ok(CrateManifest::Crate(deps))
}

1
src/parsers/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod manifest;

View file

@ -1,52 +0,0 @@
use futures::{Future, Stream, IntoFuture, future};
use hyper::{Error as HyperError, Method, Request, Response, StatusCode};
use hyper::error::UriError;
use tokio_service::Service;
use serde_json;
use ::models::crates::CrateName;
const CRATES_API_BASE_URI: &'static str = "https://crates.io/api/v1";
#[derive(Serialize, Deserialize, Debug)]
pub struct CratesVersion {
num: String,
yanked: bool
}
#[derive(Serialize, Deserialize, Debug)]
pub struct QueryCratesVersionsResponse {
versions: Vec<CratesVersion>
}
#[derive(Debug)]
pub enum QueryCratesVersionsError {
Uri(UriError),
Status(StatusCode),
Transport(HyperError),
Decode(serde_json::Error)
}
pub fn query_crates_versions<S>(service: S, crate_name: &CrateName) ->
impl Future<Item=QueryCratesVersionsResponse, Error=QueryCratesVersionsError>
where S: Service<Request=Request, Response=Response, Error=HyperError>
{
let uri_future = format!("{}/crates/{}/versions", CRATES_API_BASE_URI, crate_name.as_ref())
.parse().into_future().map_err(QueryCratesVersionsError::Uri);
uri_future.and_then(move |uri| {
let request = Request::new(Method::Get, uri);
service.call(request).map_err(QueryCratesVersionsError::Transport).and_then(|response| {
let status = response.status();
if !status.is_success() {
future::Either::A(future::err(QueryCratesVersionsError::Status(status)))
} else {
let body_future = response.body().concat2().map_err(QueryCratesVersionsError::Transport);
let decode_future = body_future
.and_then(|body| serde_json::from_slice(&body).map_err(QueryCratesVersionsError::Decode));
future::Either::B(decode_future)
}
})
})
}

View file

View file

@ -1,42 +0,0 @@
use futures::{Future, future};
use hyper::{Client, Error as HyperError, Request, Response, StatusCode};
use hyper::client::HttpConnector;
use hyper_tls::HttpsConnector;
use slog::Logger;
use tokio_service::Service;
use ::robots::crates::query_crates_versions;
pub struct Serve {
pub client: Client<HttpsConnector<HttpConnector>>,
pub logger: Logger
}
impl Service for Serve {
type Request = Request;
type Response = Response;
type Error = HyperError;
type Future = Box<Future<Item=Response, Error=HyperError>>;
fn call(&self, req: Request) -> Self::Future {
let crate_name = "hyper".parse().unwrap();
let future = query_crates_versions(self.client.clone(), &crate_name).then(|result| {
match result {
Err(err) => {
let mut response = Response::new();
response.set_status(StatusCode::InternalServerError);
response.set_body(format!("{:?}", err));
future::Either::A(future::ok(response))
},
Ok(crates_response) => {
let mut response = Response::new();
response.set_body(format!("{:?}", crates_response));
future::Either::B(future::ok(response))
}
}
});
Box::new(future)
}
}