feliix42/deps.rs
1
0
Fork 0
mirror of https://github.com/deps-rs/deps.rs.git synced 2024-11-22 02:16:30 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #44 from deps-rs/modernize

Browse source 
Modernize
This commit is contained in:
Felix Suchert 2020-09-29 20:11:59 +02:00 committed by GitHub
commit 59410cc7fe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
34 changed files with 2304 additions and 1578 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1817
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

59
Cargo.toml
View file

@ -2,35 +2,40 @@
name = "shiny-robots"
version = "0.1.0"
authors = ["Sam Rijs <srijs@airpost.net>"]
edition = "2018"
[workspace]
members = [
".",
"./libs/badge",
]
[dependencies]
cadence = "0.13.1"
failure = "0.1.1"
futures = "0.1.18"
hyper = "0.11.15"
hyper-tls = "0.1.2"
indexmap = { version = "1.0.0", features = ["serde-1"] }
lazy_static = "1.0.0"
lru-cache = "0.1.1"
maud = "0.17.2"
relative-path = { version = "0.3.7", features = ["serde"] }
route-recognizer = "0.1.12"
rustsec = "0.6.0"
semver = { version = "0.9.0", features = ["serde"] }
serde = "1.0.27"
serde_derive = "1.0.27"
serde_json = "1.0.9"
shared_failure = "0.1.0"
slog = "2.1.1"
slog-json = "2.2.0"
tokio-core = "0.1.12"
tokio-service = "0.1.0"
toml = "0.4.5"
try_future = "0.1.1"
badge = { path = "./libs/badge" }
anyhow = "1"
cadence = "0.13"
derive_more = "0.99"
futures = "0.1"
hyper = "0.11"
hyper-tls = "0.1"
indexmap = { version = "1", features = ["serde-1"] }
lru-cache = "0.1"
maud = "0.22"
once_cell = "1.4"
relative-path = { version = "0.3.7", features = ["serde"] }
route-recognizer = "0.1"
rustsec = "0.21"
semver = { version = "0.11", features = ["serde"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
slog = "2"
slog-json = "2"
tokio-core = "0.1"
tokio-service = "0.1"
toml = "0.5"
try_future = "0.1"
[dependencies.badge]
version = "0.2.0"
path = "libs/badge"
[build-dependencies]
sass-rs = "0.2.1"
sass-rs = "0.2"

3
build.rs
View file

@ -11,8 +11,7 @@ fn build_style() -> String {
..Default::default()
};
sass::compile_file("./assets/styles/main.sass", options)
.expect("failed to compile style sheet")
sass::compile_file("./assets/styles/main.sass", options).expect("failed to compile style sheet")
}
fn main() {

1
libs/badge/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
/test.svg

7
libs/badge/Cargo.toml
View file

@ -6,11 +6,12 @@ authors = ["Onur Aslan <onur@onur.im>"]
license-file = "LICENSE"
repository = "https://github.com/onur/docs.rs"
documentation = "https://docs.rs/badge"
edition = "2018"
[lib]
path = "badge.rs"
[dependencies]
base64 = "0.9.0"
lazy_static = "1.0.0"
rusttype = "0.5.0"
base64 = "0.12"
once_cell = "1"
rusttype = "0.9"

71
libs/badge/badge.rs
View file

@ -1,18 +1,16 @@
//! Simple badge generator
extern crate base64;
#[macro_use] extern crate lazy_static;
extern crate rusttype;
use base64::display::Base64Display;
use rusttype::{Font, FontCollection, Scale, point, Point, PositionedGlyph};
use once_cell::sync::Lazy;
use rusttype::{point, Font, Point, PositionedGlyph, Scale};
const FONT_DATA: &'static [u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"),
"/DejaVuSans.ttf"));
const FONT_DATA: &'static [u8] =
include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/DejaVuSans.ttf"));
const FONT_SIZE: f32 = 11.;
const SCALE: Scale = Scale {
x: FONT_SIZE,
y: FONT_SIZE,
};
pub struct BadgeOptions {
/// Subject will be displayed on the left side of badge
@ -23,7 +21,6 @@ pub struct BadgeOptions {
pub color: String,
}
impl Default for BadgeOptions {
fn default() -> BadgeOptions {
BadgeOptions {
@ -34,54 +31,47 @@ impl Default for BadgeOptions {
}
}
struct BadgeStaticData {
font: Font<'static>,
scale: Scale,
offset: Point<f32>
offset: Point<f32>,
}
static DATA: Lazy<BadgeStaticData> = Lazy::new(|| {
let font = Font::try_from_bytes(FONT_DATA).expect("failed to parse font collection");
lazy_static! {
static ref DATA: BadgeStaticData = {
let collection = FontCollection::from_bytes(FONT_DATA)
.expect("failed to parse font collection");
let font = collection.into_font()
.expect("failed to load font data");
let scale = Scale {
x: FONT_SIZE,
y: FONT_SIZE,
};
let v_metrics = font.v_metrics(scale);
let offset = point(0.0, v_metrics.ascent);
BadgeStaticData { font, scale, offset }
};
}
let v_metrics = font.v_metrics(SCALE);
let offset = point(0.0, v_metrics.ascent);
BadgeStaticData {
font,
scale: SCALE.clone(),
offset,
}
});
pub struct Badge {
options: BadgeOptions
options: BadgeOptions,
}
impl Badge {
pub fn new(options: BadgeOptions) -> Badge {
Badge { options }
}
pub fn to_svg_data_uri(&self) -> String {
format!("data:image/svg+xml;base64,{}",
Base64Display::standard(self.to_svg().as_bytes()))
format!(
"data:image/svg+xml;base64,{}",
Base64Display::with_config(self.to_svg().as_bytes(), base64::STANDARD)
)
}
pub fn to_svg(&self) -> String {
let left_width = self.calculate_width(&self.options.subject) + 6;
let right_width = self.calculate_width(&self.options.status) + 6;
let svg = format!(r###"<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="{}" height="20">
let svg = format!(
r###"<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="{}" height="20">
<linearGradient id="smooth" x2="0" y2="100%">
<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
<stop offset="1" stop-opacity=".1"/>
@ -118,16 +108,17 @@ impl Badge {
left_width + (right_width / 2),
self.options.status,
left_width + (right_width / 2),
self.options.status);
self.options.status
);
svg
}
fn calculate_width(&self, text: &str) -> u32 {
let glyphs: Vec<PositionedGlyph> =
DATA.font.layout(text, DATA.scale, DATA.offset).collect();
let width = glyphs.iter()
let width = glyphs
.iter()
.rev()
.filter_map(|g| {
g.pixel_bounding_box()
@ -139,8 +130,6 @@ impl Badge {
}
}
#[cfg(test)]
mod tests {
use super::*;

66
src/engine/fut/analyze.rs Normal file
View file

@ -0,0 +1,66 @@
use anyhow::{anyhow, ensure, Error};
use futures::stream::futures_unordered;
use futures::{Future, Poll, Stream};
use crate::models::crates::{AnalyzedDependencies, CrateDeps};
use super::super::machines::analyzer::DependencyAnalyzer;
use super::super::Engine;
pub struct AnalyzeDependenciesFuture {
inner: Box<dyn Future<Item = AnalyzedDependencies, Error = Error>>,
}
impl AnalyzeDependenciesFuture {
pub fn new(engine: Engine, deps: CrateDeps) -> Self {
let future =
engine.fetch_advisory_db().and_then(move |advisory_db| {
let analyzer = DependencyAnalyzer::new(&deps, Some(advisory_db));
let main_deps = deps.main.into_iter().filter_map(|(name, dep)| {
if dep.is_external() {
Some(name)
} else {
None
}
});
let dev_deps = deps.dev.into_iter().filter_map(|(name, dep)| {
if dep.is_external() {
Some(name)
} else {
None
}
});
let build_deps = deps.build.into_iter().filter_map(|(name, dep)| {
if dep.is_external() {
Some(name)
} else {
None
}
});
let release_futures =
engine.fetch_releases(main_deps.chain(dev_deps).chain(build_deps));
futures_unordered(release_futures)
.fold(analyzer, |mut analyzer, releases| {
analyzer.process(releases);
Ok(analyzer) as Result<_, Error>
})
.map(|analyzer| analyzer.finalize())
});
AnalyzeDependenciesFuture {
inner: Box::new(future),
}
}
}
impl Future for AnalyzeDependenciesFuture {
type Item = AnalyzedDependencies;
type Error = Error;
fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
self.inner.poll()
}
}

31
src/engine/futures/crawl.rs → src/engine/fut/crawl.rs
View file

@ -1,34 +1,40 @@
use std::mem;
use failure::Error;
use futures::{Async, Future, Poll, Stream};
use anyhow::{anyhow, ensure, Error};
use futures::stream::FuturesOrdered;
use futures::{try_ready, Async, Future, Poll, Stream};
use relative_path::RelativePathBuf;
use ::models::repo::RepoPath;
use crate::models::repo::RepoPath;
use super::super::Engine;
use super::super::machines::crawler::ManifestCrawler;
pub use super::super::machines::crawler::ManifestCrawlerOutput;
use super::super::Engine;
pub struct CrawlManifestFuture {
repo_path: RepoPath,
engine: Engine,
crawler: ManifestCrawler,
futures: FuturesOrdered<Box<Future<Item=(RelativePathBuf, String), Error=Error>>>
futures: FuturesOrdered<Box<dyn Future<Item = (RelativePathBuf, String), Error = Error>>>,
}
impl CrawlManifestFuture {
pub fn new(engine: &Engine, repo_path: RepoPath, entry_point: RelativePathBuf) -> Self {
let future: Box<Future<Item=_, Error=_>> = Box::new(engine.retrieve_manifest_at_path(&repo_path, &entry_point)
.map(move |contents| (entry_point, contents)));
let future: Box<dyn Future<Item = _, Error = _>> = Box::new(
engine
.retrieve_manifest_at_path(&repo_path, &entry_point)
.map(move |contents| (entry_point, contents)),
);
let engine = engine.clone();
let crawler = ManifestCrawler::new();
let mut futures = FuturesOrdered::new();
futures.push(future);
CrawlManifestFuture {
repo_path, engine, crawler, futures
repo_path,
engine,
crawler,
futures,
}
}
}
@ -42,12 +48,15 @@ impl Future for CrawlManifestFuture {
None => {
let crawler = mem::replace(&mut self.crawler, ManifestCrawler::new());
Ok(Async::Ready(crawler.finalize()))
},
}
Some((path, raw_manifest)) => {
let output = self.crawler.step(path, raw_manifest)?;
for path in output.paths_of_interest.into_iter() {
let future: Box<Future<Item=_, Error=_>> = Box::new(self.engine.retrieve_manifest_at_path(&self.repo_path, &path)
.map(move |contents| (path, contents)));
let future: Box<dyn Future<Item = _, Error = _>> = Box::new(
self.engine
.retrieve_manifest_at_path(&self.repo_path, &path)
.map(move |contents| (path, contents)),
);
self.futures.push(future);
}
self.poll()

4
src/engine/futures/mod.rs → src/engine/fut/mod.rs
View file

@ -1,5 +1,5 @@
mod crawl;
mod analyze;
mod crawl;
pub use self::crawl::CrawlManifestFuture;
pub use self::analyze::AnalyzeDependenciesFuture;
pub use self::crawl::CrawlManifestFuture;

49
src/engine/futures/analyze.rs
View file

@ -1,49 +0,0 @@
use failure::Error;
use futures::{Future, Poll, Stream};
use futures::stream::futures_unordered;
use ::models::crates::{AnalyzedDependencies, CrateDeps};
use super::super::Engine;
use super::super::machines::analyzer::DependencyAnalyzer;
pub struct AnalyzeDependenciesFuture {
inner: Box<Future<Item=AnalyzedDependencies, Error=Error>>
}
impl AnalyzeDependenciesFuture {
pub fn new(engine: Engine, deps: CrateDeps) -> Self {
let future = engine.fetch_advisory_db().and_then(move |advisory_db| {
let analyzer = DependencyAnalyzer::new(&deps, Some(advisory_db));
let main_deps = deps.main.into_iter().filter_map(|(name, dep)| {
if dep.is_external() { Some(name) } else { None }
});
let dev_deps = deps.dev.into_iter().filter_map(|(name, dep)| {
if dep.is_external() { Some(name) } else { None }
});
let build_deps = deps.build.into_iter().filter_map(|(name, dep)| {
if dep.is_external() { Some(name) } else { None }
});
let release_futures = engine.fetch_releases(main_deps.chain(dev_deps).chain(build_deps));
futures_unordered(release_futures)
.fold(analyzer, |mut analyzer, releases| { analyzer.process(releases); Ok(analyzer) as Result<_, Error> })
.map(|analyzer| analyzer.finalize())
});
AnalyzeDependenciesFuture {
inner: Box::new(future)
}
}
}
impl Future for AnalyzeDependenciesFuture {
type Item = AnalyzedDependencies;
type Error = Error;
fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
self.inner.poll()
}
}

179
src/engine/machines/analyzer.rs
View file

@ -1,24 +1,31 @@
use std::sync::Arc;
use rustsec::db::AdvisoryDatabase;
use rustsec::database::{Database, Query};
use semver::Version;
use ::models::crates::{CrateDeps, CrateRelease, CrateName, AnalyzedDependency, AnalyzedDependencies};
use crate::models::crates::{
AnalyzedDependencies, AnalyzedDependency, CrateDeps, CrateName, CrateRelease,
};
pub struct DependencyAnalyzer {
deps: AnalyzedDependencies,
advisory_db: Option<Arc<AdvisoryDatabase>>
advisory_db: Option<Arc<Database>>,
}
impl DependencyAnalyzer {
pub fn new(deps: &CrateDeps, advisory_db: Option<Arc<AdvisoryDatabase>>) -> DependencyAnalyzer {
pub fn new(deps: &CrateDeps, advisory_db: Option<Arc<Database>>) -> DependencyAnalyzer {
DependencyAnalyzer {
deps: AnalyzedDependencies::new(deps),
advisory_db
advisory_db,
}
}
fn process_single(name: &CrateName, dep: &mut AnalyzedDependency, ver: &Version, advisory_db: Option<&AdvisoryDatabase>) {
fn process_single(
name: &CrateName,
dep: &mut AnalyzedDependency,
ver: &Version,
advisory_db: Option<&Database>,
) {
if dep.required.matches(&ver) {
if let Some(ref mut current_latest_that_matches) = dep.latest_that_matches {
if *current_latest_that_matches < *ver {
@ -28,7 +35,14 @@ impl DependencyAnalyzer {
dep.latest_that_matches = Some(ver.clone());
}
if !advisory_db.map(|db| db.find_vulns_for_crate(name.as_ref(), ver).is_empty()).unwrap_or(true) {
let name: rustsec::cargo_lock::Name = name.as_ref().parse().unwrap();
let version: rustsec::cargo_lock::Version = ver.to_string().parse().unwrap();
let query = Query::new().package_version(name, version);
if !advisory_db
.map(|db| db.query(&query).is_empty())
.unwrap_or(true)
{
dep.insecure = true;
}
}
@ -43,17 +57,32 @@ impl DependencyAnalyzer {
}
}
pub fn process<I: IntoIterator<Item=CrateRelease>>(&mut self, releases: I) {
pub fn process<I: IntoIterator<Item = CrateRelease>>(&mut self, releases: I) {
let advisory_db = self.advisory_db.as_ref().map(|r| r.as_ref());
for release in releases.into_iter().filter(|r| !r.yanked) {
if let Some(main_dep) = self.deps.main.get_mut(&release.name) {
DependencyAnalyzer::process_single(&release.name, main_dep, &release.version, advisory_db)
DependencyAnalyzer::process_single(
&release.name,
main_dep,
&release.version,
advisory_db,
)
}
if let Some(dev_dep) = self.deps.dev.get_mut(&release.name) {
DependencyAnalyzer::process_single(&release.name, dev_dep, &release.version, advisory_db)
DependencyAnalyzer::process_single(
&release.name,
dev_dep,
&release.version,
advisory_db,
)
}
if let Some(build_dep) = self.deps.build.get_mut(&release.name) {
DependencyAnalyzer::process_single(&release.name, build_dep, &release.version, advisory_db)
DependencyAnalyzer::process_single(
&release.name,
build_dep,
&release.version,
advisory_db,
)
}
}
}
@ -65,75 +94,157 @@ impl DependencyAnalyzer {
#[cfg(test)]
mod tests {
use models::crates::{CrateDep, CrateDeps, CrateRelease};
use super::DependencyAnalyzer;
use crate::models::crates::{CrateDep, CrateDeps, CrateRelease};
use super::*;
#[test]
fn tracks_latest_without_matching() {
let mut deps = CrateDeps::default();
deps.main.insert("hyper".parse().unwrap(), CrateDep::External("^0.11.0".parse().unwrap()));
deps.main.insert(
"hyper".parse().unwrap(),
CrateDep::External("^0.11.0".parse().unwrap()),
);
let mut analyzer = DependencyAnalyzer::new(&deps, None);
analyzer.process(vec![
CrateRelease { name: "hyper".parse().unwrap(), version: "0.10.0".parse().unwrap(), deps: Default::default(), yanked: false },
CrateRelease { name: "hyper".parse().unwrap(), version: "0.10.1".parse().unwrap(), deps: Default::default(), yanked: false }
CrateRelease {
name: "hyper".parse().unwrap(),
version: "0.10.0".parse().unwrap(),
deps: Default::default(),
yanked: false,
},
CrateRelease {
name: "hyper".parse().unwrap(),
version: "0.10.1".parse().unwrap(),
deps: Default::default(),
yanked: false,
},
]);
let analyzed = analyzer.finalize();
assert_eq!(analyzed.main.get("hyper").unwrap().latest_that_matches, None);
assert_eq!(analyzed.main.get("hyper").unwrap().latest, Some("0.10.1".parse().unwrap()));
assert_eq!(
analyzed.main.get("hyper").unwrap().latest_that_matches,
None
);
assert_eq!(
analyzed.main.get("hyper").unwrap().latest,
Some("0.10.1".parse().unwrap())
);
}
#[test]
fn tracks_latest_that_matches() {
let mut deps = CrateDeps::default();
deps.main.insert("hyper".parse().unwrap(), CrateDep::External("^0.10.0".parse().unwrap()));
deps.main.insert(
"hyper".parse().unwrap(),
CrateDep::External("^0.10.0".parse().unwrap()),
);
let mut analyzer = DependencyAnalyzer::new(&deps, None);
analyzer.process(vec![
CrateRelease { name: "hyper".parse().unwrap(), version: "0.10.0".parse().unwrap(), deps: Default::default(), yanked: false },
CrateRelease { name: "hyper".parse().unwrap(), version: "0.10.1".parse().unwrap(), deps: Default::default(), yanked: false },
CrateRelease { name: "hyper".parse().unwrap(), version: "0.11.0".parse().unwrap(), deps: Default::default(), yanked: false }
CrateRelease {
name: "hyper".parse().unwrap(),
version: "0.10.0".parse().unwrap(),
deps: Default::default(),
yanked: false,
},
CrateRelease {
name: "hyper".parse().unwrap(),
version: "0.10.1".parse().unwrap(),
deps: Default::default(),
yanked: false,
},
CrateRelease {
name: "hyper".parse().unwrap(),
version: "0.11.0".parse().unwrap(),
deps: Default::default(),
yanked: false,
},
]);
let analyzed = analyzer.finalize();
assert_eq!(analyzed.main.get("hyper").unwrap().latest_that_matches, Some("0.10.1".parse().unwrap()));
assert_eq!(analyzed.main.get("hyper").unwrap().latest, Some("0.11.0".parse().unwrap()));
assert_eq!(
analyzed.main.get("hyper").unwrap().latest_that_matches,
Some("0.10.1".parse().unwrap())
);
assert_eq!(
analyzed.main.get("hyper").unwrap().latest,
Some("0.11.0".parse().unwrap())
);
}
#[test]
fn skips_yanked_releases() {
let mut deps = CrateDeps::default();
deps.main.insert("hyper".parse().unwrap(), CrateDep::External("^0.10.0".parse().unwrap()));
deps.main.insert(
"hyper".parse().unwrap(),
CrateDep::External("^0.10.0".parse().unwrap()),
);
let mut analyzer = DependencyAnalyzer::new(&deps, None);
analyzer.process(vec![
CrateRelease { name: "hyper".parse().unwrap(), version: "0.10.0".parse().unwrap(), deps: Default::default(), yanked: false },
CrateRelease { name: "hyper".parse().unwrap(), version: "0.10.1".parse().unwrap(), deps: Default::default(), yanked: true },
CrateRelease {
name: "hyper".parse().unwrap(),
version: "0.10.0".parse().unwrap(),
deps: Default::default(),
yanked: false,
},
CrateRelease {
name: "hyper".parse().unwrap(),
version: "0.10.1".parse().unwrap(),
deps: Default::default(),
yanked: true,
},
]);
let analyzed = analyzer.finalize();
assert_eq!(analyzed.main.get("hyper").unwrap().latest_that_matches, Some("0.10.0".parse().unwrap()));
assert_eq!(analyzed.main.get("hyper").unwrap().latest, Some("0.10.0".parse().unwrap()));
assert_eq!(
analyzed.main.get("hyper").unwrap().latest_that_matches,
Some("0.10.0".parse().unwrap())
);
assert_eq!(
analyzed.main.get("hyper").unwrap().latest,
Some("0.10.0".parse().unwrap())
);
}
#[test]
fn skips_prereleases() {
let mut deps = CrateDeps::default();
deps.main.insert("hyper".parse().unwrap(), CrateDep::External("^0.10.0".parse().unwrap()));
deps.main.insert(
"hyper".parse().unwrap(),
CrateDep::External("^0.10.0".parse().unwrap()),
);
let mut analyzer = DependencyAnalyzer::new(&deps, None);
analyzer.process(vec![
CrateRelease { name: "hyper".parse().unwrap(), version: "0.10.0".parse().unwrap(), deps: Default::default(), yanked: false },
CrateRelease { name: "hyper".parse().unwrap(), version: "0.10.1-alpha".parse().unwrap(), deps: Default::default(), yanked: false },
CrateRelease {
name: "hyper".parse().unwrap(),
version: "0.10.0".parse().unwrap(),
deps: Default::default(),
yanked: false,
},
CrateRelease {
name: "hyper".parse().unwrap(),
version: "0.10.1-alpha".parse().unwrap(),
deps: Default::default(),
yanked: false,
},
]);
let analyzed = analyzer.finalize();
assert_eq!(analyzed.main.get("hyper").unwrap().latest_that_matches, Some("0.10.0".parse().unwrap()));
assert_eq!(analyzed.main.get("hyper").unwrap().latest, Some("0.10.0".parse().unwrap()));
assert_eq!(
analyzed.main.get("hyper").unwrap().latest_that_matches,
Some("0.10.0".parse().unwrap())
);
assert_eq!(
analyzed.main.get("hyper").unwrap().latest,
Some("0.10.0".parse().unwrap())
);
}
}

129
src/engine/machines/crawler.rs
View file

@ -1,49 +1,57 @@
use std::collections::HashMap;
use failure::Error;
use relative_path::RelativePathBuf;
use anyhow::{anyhow, ensure, Error};
use indexmap::IndexMap;
use relative_path::RelativePathBuf;
use ::parsers::manifest::parse_manifest_toml;
use ::models::crates::{CrateDep, CrateDeps, CrateName, CrateManifest};
use crate::models::crates::{CrateDep, CrateDeps, CrateManifest, CrateName};
use crate::parsers::manifest::parse_manifest_toml;
pub struct ManifestCrawlerOutput {
pub crates: IndexMap<CrateName, CrateDeps>
pub crates: IndexMap<CrateName, CrateDeps>,
}
pub struct ManifestCrawlerStepOutput {
pub paths_of_interest: Vec<RelativePathBuf>
pub paths_of_interest: Vec<RelativePathBuf>,
}
pub struct ManifestCrawler {
manifests: HashMap<RelativePathBuf, CrateManifest>,
leaf_crates: IndexMap<CrateName, CrateDeps>
leaf_crates: IndexMap<CrateName, CrateDeps>,
}
impl ManifestCrawler {
pub fn new() -> ManifestCrawler {
ManifestCrawler {
manifests: HashMap::new(),
leaf_crates: IndexMap::new()
leaf_crates: IndexMap::new(),
}
}
pub fn step(&mut self, path: RelativePathBuf, raw_manifest: String) -> Result<ManifestCrawlerStepOutput, Error> {
pub fn step(
&mut self,
path: RelativePathBuf,
raw_manifest: String,
) -> Result<ManifestCrawlerStepOutput, Error> {
let manifest = parse_manifest_toml(&raw_manifest)?;
self.manifests.insert(path.clone(), manifest.clone());
let mut output = ManifestCrawlerStepOutput {
paths_of_interest: vec![]
paths_of_interest: vec![],
};
match manifest {
CrateManifest::Package(name, deps) => {
self.process_package(&path, name, deps, &mut output);
},
}
CrateManifest::Workspace { members } => {
self.process_workspace(&path, &members, &mut output);
},
CrateManifest::Mixed { name, deps, members } => {
}
CrateManifest::Mixed {
name,
deps,
members,
} => {
self.process_package(&path, name, deps, &mut output);
self.process_workspace(&path, &members, &mut output);
}
@ -52,15 +60,31 @@ impl ManifestCrawler {
Ok(output)
}
fn register_interest(&mut self, base_path: &RelativePathBuf, path: &RelativePathBuf, output: &mut ManifestCrawlerStepOutput) {
fn register_interest(
&mut self,
base_path: &RelativePathBuf,
path: &RelativePathBuf,
output: &mut ManifestCrawlerStepOutput,
) {
let full_path = base_path.join_normalized(path);
if !self.manifests.contains_key(&full_path) {
output.paths_of_interest.push(full_path);
}
}
fn process_package(&mut self, base_path: &RelativePathBuf, name: CrateName, deps: CrateDeps, output: &mut ManifestCrawlerStepOutput) {
for (_, dep) in deps.main.iter().chain(deps.dev.iter()).chain(deps.build.iter()) {
fn process_package(
&mut self,
base_path: &RelativePathBuf,
name: CrateName,
deps: CrateDeps,
output: &mut ManifestCrawlerStepOutput,
) {
for (_, dep) in deps
.main
.iter()
.chain(deps.dev.iter())
.chain(deps.build.iter())
{
if let &CrateDep::Internal(ref path) = dep {
self.register_interest(base_path, path, output);
}
@ -69,8 +93,13 @@ impl ManifestCrawler {
self.leaf_crates.insert(name, deps);
}
fn process_workspace(&mut self, base_path: &RelativePathBuf, members: &[RelativePathBuf], output: &mut ManifestCrawlerStepOutput) {
for mut path in members {
fn process_workspace(
&mut self,
base_path: &RelativePathBuf,
members: &[RelativePathBuf],
output: &mut ManifestCrawlerStepOutput,
) {
for path in members {
if !path.ends_with("*") {
self.register_interest(base_path, path, output);
}
@ -79,7 +108,7 @@ impl ManifestCrawler {
pub fn finalize(self) -> ManifestCrawlerOutput {
ManifestCrawlerOutput {
crates: self.leaf_crates
crates: self.leaf_crates,
}
}
}
@ -89,8 +118,9 @@ mod tests {
use relative_path::RelativePath;
use semver::VersionReq;
use models::crates::CrateDep;
use super::ManifestCrawler;
use crate::models::crates::CrateDep;
use super::*;
#[test]
fn simple_package_manifest() {
@ -99,7 +129,9 @@ mod tests {
name = "simpleton"
"#;
let mut crawler = ManifestCrawler::new();
let step_output = crawler.step("Cargo.toml".into(), manifest.to_string()).unwrap();
let step_output = crawler
.step("Cargo.toml".into(), manifest.to_string())
.unwrap();
assert_eq!(step_output.paths_of_interest.len(), 0);
let output = crawler.finalize();
assert_eq!(output.crates.len(), 1);
@ -127,16 +159,24 @@ codegen = "0.0.1"
let output = crawler.finalize();
assert_eq!(output.crates.len(), 1);
assert_eq!(output.crates["more-complex"].main.len(), 2);
assert_eq!(output.crates["more-complex"].main.get("foo").unwrap(),
&CrateDep::External(VersionReq::parse("0.30.0").unwrap()));
assert_eq!(output.crates["more-complex"].main.get("bar").unwrap(),
&CrateDep::External(VersionReq::parse("1.2.0").unwrap()));
assert_eq!(
output.crates["more-complex"].main.get("foo").unwrap(),
&CrateDep::External(VersionReq::parse("0.30.0").unwrap())
);
assert_eq!(
output.crates["more-complex"].main.get("bar").unwrap(),
&CrateDep::External(VersionReq::parse("1.2.0").unwrap())
);
assert_eq!(output.crates["more-complex"].dev.len(), 1);
assert_eq!(output.crates["more-complex"].dev.get("quickcheck").unwrap(),
&CrateDep::External(VersionReq::parse("0.5").unwrap()));
assert_eq!(
output.crates["more-complex"].dev.get("quickcheck").unwrap(),
&CrateDep::External(VersionReq::parse("0.5").unwrap())
);
assert_eq!(output.crates["more-complex"].build.len(), 1);
assert_eq!(output.crates["more-complex"].build.get("codegen").unwrap(),
&CrateDep::External(VersionReq::parse("0.0.1").unwrap()));
assert_eq!(
output.crates["more-complex"].build.get("codegen").unwrap(),
&CrateDep::External(VersionReq::parse("0.0.1").unwrap())
);
}
#[test]
@ -226,10 +266,17 @@ features = ["use_std"]
"#;
let mut crawler = ManifestCrawler::new();
let step_output = crawler.step("".into(), futures_manifest.to_string()).unwrap();
let step_output = crawler
.step("".into(), futures_manifest.to_string())
.unwrap();
assert_eq!(step_output.paths_of_interest.len(), 1);
assert_eq!(step_output.paths_of_interest[0].as_str(), "futures-cpupool");
let step_output = crawler.step("futures-cpupool".into(), futures_cpupool_manifest.to_string()).unwrap();
let step_output = crawler
.step(
"futures-cpupool".into(),
futures_cpupool_manifest.to_string(),
)
.unwrap();
assert_eq!(step_output.paths_of_interest.len(), 0);
let output = crawler.finalize();
assert_eq!(output.crates.len(), 2);
@ -237,10 +284,20 @@ features = ["use_std"]
assert_eq!(output.crates["futures"].dev.len(), 0);
assert_eq!(output.crates["futures"].build.len(), 0);
assert_eq!(output.crates["futures-cpupool"].main.len(), 2);
assert_eq!(output.crates["futures-cpupool"].main.get("num_cpus").unwrap(),
&CrateDep::External(VersionReq::parse("1.0").unwrap()));
assert_eq!(output.crates["futures-cpupool"].main.get("futures").unwrap(),
&CrateDep::Internal(RelativePath::new("..").to_relative_path_buf()));
assert_eq!(
output.crates["futures-cpupool"]
.main
.get("num_cpus")
.unwrap(),
&CrateDep::External(VersionReq::parse("1.0").unwrap())
);
assert_eq!(
output.crates["futures-cpupool"]
.main
.get("futures")
.unwrap(),
&CrateDep::Internal(RelativePath::new("..").to_relative_path_buf())
);
assert_eq!(output.crates["futures-cpupool"].dev.len(), 0);
assert_eq!(output.crates["futures-cpupool"].build.len(), 0);
}

2
src/engine/machines/mod.rs
View file

@ -1,2 +1,2 @@
pub mod crawler;
pub mod analyzer;
pub mod crawler;

229
src/engine/mod.rs
View file

@ -4,33 +4,34 @@ use std::time::{Duration, Instant};
use cadence::prelude::*;
use cadence::{MetricSink, NopMetricSink, StatsdClient};
use failure::Error;
use futures::{Future, future};
use anyhow::{anyhow, ensure, Error};
use futures::future::join_all;
use hyper::Client;
use futures::{future, Future};
use hyper::client::HttpConnector;
use hyper::Client;
use hyper_tls::HttpsConnector;
use once_cell::sync::Lazy;
use relative_path::{RelativePath, RelativePathBuf};
use rustsec::db::AdvisoryDatabase;
use rustsec::database::Database;
use semver::VersionReq;
use slog::Logger;
use tokio_service::Service;
use crate::utils::cache::Cache;
use crate::models::crates::{AnalyzedDependencies, CrateName, CratePath, CrateRelease};
use crate::models::repo::{RepoPath, Repository};
use crate::interactors::crates::{GetPopularCrates, QueryCrate};
use crate::interactors::github::GetPopularRepos;
use crate::interactors::rustsec::FetchAdvisoryDatabase;
use crate::interactors::RetrieveFileAtPath;
mod fut;
mod machines;
mod futures;
use ::utils::cache::Cache;
use ::models::repo::{Repository, RepoPath};
use ::models::crates::{CrateName, CratePath, CrateRelease, AnalyzedDependencies};
use ::interactors::crates::{QueryCrate, GetPopularCrates};
use ::interactors::RetrieveFileAtPath;
use ::interactors::github::GetPopularRepos;
use ::interactors::rustsec::FetchAdvisoryDatabase;
use self::futures::AnalyzeDependenciesFuture;
use self::futures::CrawlManifestFuture;
use self::fut::AnalyzeDependenciesFuture;
use self::fut::CrawlManifestFuture;
type HttpClient = Client<HttpsConnector<HttpConnector>>;
@ -44,7 +45,7 @@ pub struct Engine {
get_popular_crates: Arc<Cache<GetPopularCrates<HttpClient>>>,
get_popular_repos: Arc<Cache<GetPopularRepos<HttpClient>>>,
retrieve_file_at_path: Arc<RetrieveFileAtPath<HttpClient>>,
fetch_advisory_db: Arc<Cache<FetchAdvisoryDatabase<HttpClient>>>
fetch_advisory_db: Arc<Cache<FetchAdvisoryDatabase<HttpClient>>>,
}
impl Engine {
@ -52,18 +53,26 @@ impl Engine {
let metrics = StatsdClient::from_sink("engine", NopMetricSink);
let query_crate = Cache::new(QueryCrate(client.clone()), Duration::from_secs(300), 500);
let get_popular_crates = Cache::new(GetPopularCrates(client.clone()), Duration::from_secs(10), 1);
let get_popular_repos = Cache::new(GetPopularRepos(client.clone()), Duration::from_secs(10), 1);
let fetch_advisory_db = Cache::new(FetchAdvisoryDatabase(client.clone()), Duration::from_secs(300), 1);
let get_popular_crates =
Cache::new(GetPopularCrates(client.clone()), Duration::from_secs(10), 1);
let get_popular_repos =
Cache::new(GetPopularRepos(client.clone()), Duration::from_secs(10), 1);
let fetch_advisory_db = Cache::new(
FetchAdvisoryDatabase(client.clone()),
Duration::from_secs(300),
1,
);
Engine {
client: client.clone(), logger, metrics,
client: client.clone(),
logger,
metrics,
query_crate: Arc::new(query_crate),
get_popular_crates: Arc::new(get_popular_crates),
get_popular_repos: Arc::new(get_popular_repos),
retrieve_file_at_path: Arc::new(RetrieveFileAtPath(client)),
fetch_advisory_db: Arc::new(fetch_advisory_db)
fetch_advisory_db: Arc::new(fetch_advisory_db),
}
}
@ -74,7 +83,7 @@ impl Engine {
pub struct AnalyzeDependenciesOutcome {
pub crates: Vec<(CrateName, AnalyzedDependencies)>,
pub duration: Duration
pub duration: Duration,
}
impl AnalyzeDependenciesOutcome {
@ -83,38 +92,42 @@ impl AnalyzeDependenciesOutcome {
}
pub fn any_insecure(&self) -> bool {
self.crates.iter().any(|&(_, ref deps)| deps.count_insecure() > 0)
self.crates
.iter()
.any(|&(_, ref deps)| deps.count_insecure() > 0)
}
pub fn outdated_ratio(&self) -> (usize, usize) {
self.crates.iter().fold((0, 0), |(outdated, total), &(_, ref deps)| {
(outdated + deps.count_outdated(), total + deps.count_total())
})
self.crates
.iter()
.fold((0, 0), |(outdated, total), &(_, ref deps)| {
(outdated + deps.count_outdated(), total + deps.count_total())
})
}
}
impl Engine {
pub fn get_popular_repos(&self) ->
impl Future<Item=Vec<Repository>, Error=Error>
{
self.get_popular_repos.call(())
.from_err().map(|repos| {
repos.iter()
.filter(|repo| !POPULAR_REPOS_BLACKLIST.contains(&repo.path))
.cloned().collect()
})
pub fn get_popular_repos(&self) -> impl Future<Item = Vec<Repository>, Error = Error> {
self.get_popular_repos.call(()).from_err().map(|repos| {
repos
.iter()
.filter(|repo| !POPULAR_REPO_BLOCK_LIST.contains(&repo.path))
.cloned()
.collect()
})
}
pub fn get_popular_crates(&self) ->
impl Future<Item=Vec<CratePath>, Error=Error>
{
self.get_popular_crates.call(())
.from_err().map(|crates| crates.clone())
pub fn get_popular_crates(&self) -> impl Future<Item = Vec<CratePath>, Error = Error> {
self.get_popular_crates
.call(())
.from_err()
.map(|crates| crates.clone())
}
pub fn analyze_repo_dependencies(&self, repo_path: RepoPath) ->
impl Future<Item=AnalyzeDependenciesOutcome, Error=Error>
{
pub fn analyze_repo_dependencies(
&self,
repo_path: RepoPath,
) -> impl Future<Item = AnalyzeDependenciesOutcome, Error = Error> {
let start = Instant::now();
let entry_point = RelativePath::new("/").to_relative_path_buf();
@ -123,99 +136,125 @@ impl Engine {
let engine = self.clone();
manifest_future.and_then(move |manifest_output| {
let engine_for_analyze = engine.clone();
let futures = manifest_output.crates.into_iter().map(move |(crate_name, deps)| {
let analyzed_deps_future = AnalyzeDependenciesFuture::new(engine_for_analyze.clone(), deps);
let futures = manifest_output
.crates
.into_iter()
.map(move |(crate_name, deps)| {
let analyzed_deps_future =
AnalyzeDependenciesFuture::new(engine_for_analyze.clone(), deps);
analyzed_deps_future.map(move |analyzed_deps| (crate_name, analyzed_deps))
});
analyzed_deps_future.map(move |analyzed_deps| (crate_name, analyzed_deps))
});
join_all(futures).and_then(move |crates| {
let duration = start.elapsed();
engine.metrics.time_duration_with_tags("analyze_duration", duration)
engine
.metrics
.time_duration_with_tags("analyze_duration", duration)
.with_tag("repo_site", repo_path.site.as_ref())
.with_tag("repo_qual", repo_path.qual.as_ref())
.with_tag("repo_name", repo_path.name.as_ref())
.send()?;
Ok(AnalyzeDependenciesOutcome {
crates, duration
})
Ok(AnalyzeDependenciesOutcome { crates, duration })
})
})
}
pub fn analyze_crate_dependencies(&self, crate_path: CratePath) ->
impl Future<Item=AnalyzeDependenciesOutcome, Error=Error>
{
pub fn analyze_crate_dependencies(
&self,
crate_path: CratePath,
) -> impl Future<Item = AnalyzeDependenciesOutcome, Error = Error> {
let start = Instant::now();
let query_future = self.query_crate.call(crate_path.name.clone()).from_err();
let engine = self.clone();
query_future.and_then(move |query_response| {
match query_response.releases.iter().find(|release| release.version == crate_path.version) {
None => future::Either::A(future::err(format_err!("could not find crate release with version {}", crate_path.version))),
match query_response
.releases
.iter()
.find(|release| release.version == crate_path.version)
{
None => future::Either::A(future::err(anyhow!(
"could not find crate release with version {}",
crate_path.version
))),
Some(release) => {
let analyzed_deps_future = AnalyzeDependenciesFuture::new(engine.clone(), release.deps.clone());
let analyzed_deps_future =
AnalyzeDependenciesFuture::new(engine.clone(), release.deps.clone());
future::Either::B(analyzed_deps_future.map(move |analyzed_deps| {
let crates = vec![(crate_path.name, analyzed_deps)].into_iter().collect();
let duration = start.elapsed();
AnalyzeDependenciesOutcome {
crates, duration
}
AnalyzeDependenciesOutcome { crates, duration }
}))
}
}
})
}
pub fn find_latest_crate_release(&self, name: CrateName, req: VersionReq) ->
impl Future<Item=Option<CrateRelease>, Error=Error>
{
self.query_crate.call(name).from_err().map(move |query_response| {
query_response.releases.iter()
.filter(|release| req.matches(&release.version))
.max_by(|r1, r2| r1.version.cmp(&r2.version))
.cloned()
})
pub fn find_latest_crate_release(
&self,
name: CrateName,
req: VersionReq,
) -> impl Future<Item = Option<CrateRelease>, Error = Error> {
self.query_crate
.call(name)
.from_err()
.map(move |query_response| {
query_response
.releases
.iter()
.filter(|release| req.matches(&release.version))
.max_by(|r1, r2| r1.version.cmp(&r2.version))
.cloned()
})
}
fn fetch_releases<I: IntoIterator<Item=CrateName>>(&self, names: I) ->
impl Iterator<Item=impl Future<Item=Vec<CrateRelease>, Error=Error>>
{
fn fetch_releases<I: IntoIterator<Item = CrateName>>(
&self,
names: I,
) -> impl Iterator<Item = impl Future<Item = Vec<CrateRelease>, Error = Error>> {
let engine = self.clone();
names.into_iter().map(move |name| {
engine.query_crate.call(name)
engine
.query_crate
.call(name)
.from_err()
.map(|resp| resp.releases.clone())
})
}
fn retrieve_manifest_at_path(&self, repo_path: &RepoPath, path: &RelativePathBuf) ->
impl Future<Item=String, Error=Error>
{
fn retrieve_manifest_at_path(
&self,
repo_path: &RepoPath,
path: &RelativePathBuf,
) -> impl Future<Item = String, Error = Error> {
let manifest_path = path.join(RelativePath::new("Cargo.toml"));
self.retrieve_file_at_path.call((repo_path.clone(), manifest_path))
self.retrieve_file_at_path
.call((repo_path.clone(), manifest_path))
}
fn fetch_advisory_db(&self) ->
impl Future<Item=Arc<AdvisoryDatabase>, Error=Error>
{
self.fetch_advisory_db.call(()).from_err().map(|db| db.clone())
fn fetch_advisory_db(&self) -> impl Future<Item = Arc<Database>, Error = Error> {
self.fetch_advisory_db
.call(())
.from_err()
.map(|db| db.clone())
}
}
lazy_static! {
static ref POPULAR_REPOS_BLACKLIST: HashSet<RepoPath> = {
vec![
RepoPath::from_parts("github", "rust-lang", "rust"),
RepoPath::from_parts("github", "google", "xi-editor"),
RepoPath::from_parts("github", "lk-geimfari", "awesomo"),
RepoPath::from_parts("github", "redox-os", "tfs"),
RepoPath::from_parts("github", "carols10cents", "rustlings"),
RepoPath::from_parts("github", "rust-unofficial", "awesome-rust")
].into_iter().collect::<Result<HashSet<_>, _>>().unwrap()
};
}
static POPULAR_REPO_BLOCK_LIST: Lazy<HashSet<RepoPath>> = Lazy::new(|| {
vec![
RepoPath::from_parts("github", "rust-lang", "rust"),
RepoPath::from_parts("github", "google", "xi-editor"),
RepoPath::from_parts("github", "lk-geimfari", "awesomo"),
RepoPath::from_parts("github", "redox-os", "tfs"),
RepoPath::from_parts("github", "carols10cents", "rustlings"),
RepoPath::from_parts("github", "rust-unofficial", "awesome-rust"),
]
.into_iter()
.collect::<Result<HashSet<_>, _>>()
.unwrap()
});

11
src/interactors/bitbucket.rs
View file

@ -1,18 +1,19 @@
use failure::Error;
use anyhow::{anyhow, ensure, Error};
use hyper::Uri;
use relative_path::RelativePathBuf;
use ::models::repo::RepoPath;
use crate::models::repo::RepoPath;
const BITBUCKET_USER_CONTENT_BASE_URI: &'static str = "https://bitbucket.org";
pub fn get_manifest_uri(repo_path: &RepoPath, path: &RelativePathBuf) -> Result<Uri, Error> {
let path_str: &str = path.as_ref();
Ok(format!("{}/{}/{}/raw/HEAD/{}",
Ok(format!(
"{}/{}/{}/raw/HEAD/{}",
BITBUCKET_USER_CONTENT_BASE_URI,
repo_path.qual.as_ref(),
repo_path.name.as_ref(),
path_str
).parse::<Uri>()?)
)
.parse::<Uri>()?)
}

123
src/interactors/crates.rs
View file

@ -1,13 +1,13 @@
use std::str;
use failure::Error;
use futures::{Future, Stream, IntoFuture, future};
use anyhow::{anyhow, ensure, Error};
use futures::{future, Future, IntoFuture, Stream};
use hyper::{Error as HyperError, Method, Request, Response, Uri};
use tokio_service::Service;
use semver::{Version, VersionReq};
use serde_json;
use serde::Deserialize;
use tokio_service::Service;
use ::models::crates::{CrateName, CrateRelease, CrateDeps, CrateDep, CratePath};
use crate::models::crates::{CrateDep, CrateDeps, CrateName, CratePath, CrateRelease};
const CRATES_INDEX_BASE_URI: &str = "https://raw.githubusercontent.com/rust-lang/crates.io-index";
const CRATES_API_BASE_URI: &str = "https://crates.io/api/v1";
@ -17,7 +17,7 @@ struct RegistryPackageDep {
name: String,
req: VersionReq,
#[serde(default)]
kind: Option<String>
kind: Option<String>,
}
#[derive(Deserialize, Debug)]
@ -26,51 +26,63 @@ struct RegistryPackage {
#[serde(default)]
deps: Vec<RegistryPackageDep>,
#[serde(default)]
yanked: bool
yanked: bool,
}
fn convert_pkgs(name: &CrateName, packages: Vec<RegistryPackage>) -> Result<QueryCrateResponse, Error> {
let releases = packages.into_iter().map(|package| {
let mut deps = CrateDeps::default();
for dep in package.deps {
match dep.kind.map(|k| k.clone()).unwrap_or_else(|| "normal".into()).as_ref() {
"normal" =>
deps.main.insert(dep.name.parse()?, CrateDep::External(dep.req)),
"dev" =>
deps.dev.insert(dep.name.parse()?, CrateDep::External(dep.req)),
_ => None
};
}
Ok(CrateRelease {
name: name.clone(),
version: package.vers,
deps: deps,
yanked: package.yanked
fn convert_pkgs(
name: &CrateName,
packages: Vec<RegistryPackage>,
) -> Result<QueryCrateResponse, Error> {
let releases = packages
.into_iter()
.map(|package| {
let mut deps = CrateDeps::default();
for dep in package.deps {
match dep
.kind
.map(|k| k.clone())
.unwrap_or_else(|| "normal".into())
.as_ref()
{
"normal" => deps
.main
.insert(dep.name.parse()?, CrateDep::External(dep.req)),
"dev" => deps
.dev
.insert(dep.name.parse()?, CrateDep::External(dep.req)),
_ => None,
};
}
Ok(CrateRelease {
name: name.clone(),
version: package.vers,
deps: deps,
yanked: package.yanked,
})
})
}).collect::<Result<_, Error>>()?;
.collect::<Result<_, Error>>()?;
Ok(QueryCrateResponse {
releases: releases
})
Ok(QueryCrateResponse { releases: releases })
}
pub struct QueryCrateResponse {
pub releases: Vec<CrateRelease>
pub releases: Vec<CrateRelease>,
}
#[derive(Debug, Clone)]
pub struct QueryCrate<S>(pub S);
impl<S> Service for QueryCrate<S>
where S: Service<Request=Request, Response=Response, Error=HyperError> + Clone + 'static,
S::Future: 'static
where
S: Service<Request = Request, Response = Response, Error = HyperError> + Clone + 'static,
S::Future: 'static,
{
type Request = CrateName;
type Response = QueryCrateResponse;
type Error = Error;
type Future = Box<Future<Item=Self::Response, Error=Self::Error>>;
type Future = Box<dyn Future<Item = Self::Response, Error = Self::Error>>;
fn call(&self, crate_name: CrateName) -> Self::Future {
fn call(&self, crate_name: CrateName) -> Self::Future {
let lower_name = crate_name.as_ref().to_lowercase();
let path = match lower_name.len() {
@ -80,21 +92,22 @@ impl<S> Service for QueryCrate<S>
_ => format!("{}/{}/{}", &lower_name[0..2], &lower_name[2..4], lower_name),
};
let uri = try_future_box!(format!("{}/master/{}", CRATES_INDEX_BASE_URI, path)
.parse::<Uri>());
let uri =
try_future_box!(format!("{}/master/{}", CRATES_INDEX_BASE_URI, path).parse::<Uri>());
let request = Request::new(Method::Get, uri.clone());
Box::new(self.0.call(request).from_err().and_then(move |response| {
let status = response.status();
if !status.is_success() {
try_future!(Err(format_err!("Status code {} for URI {}", status, uri)));
try_future!(Err(anyhow!("Status code {} for URI {}", status, uri)));
}
let body_future = response.body().concat2().from_err();
let decode_future = body_future.and_then(move |body| {
let string_body = str::from_utf8(body.as_ref())?;
let packages = string_body.lines()
let packages = string_body
.lines()
.map(|s| s.trim())
.filter(|s| !s.is_empty())
.map(|s| serde_json::from_str::<RegistryPackage>(s))
@ -112,38 +125,48 @@ impl<S> Service for QueryCrate<S>
#[derive(Deserialize)]
struct SummaryResponseDetail {
name: String,
max_version: Version
max_version: Version,
}
#[derive(Deserialize)]
struct SummaryResponse {
most_downloaded: Vec<SummaryResponseDetail>
most_downloaded: Vec<SummaryResponseDetail>,
}
fn convert_summary(response: SummaryResponse) -> Result<Vec<CratePath>, Error> {
response.most_downloaded.into_iter().map(|detail| {
let name = detail.name.parse()?;
Ok(CratePath { name, version: detail.max_version })
}).collect()
response
.most_downloaded
.into_iter()
.map(|detail| {
let name = detail.name.parse()?;
Ok(CratePath {
name,
version: detail.max_version,
})
})
.collect()
}
#[derive(Debug, Clone)]
pub struct GetPopularCrates<S>(pub S);
impl<S> Service for GetPopularCrates<S>
where S: Service<Request=Request, Response=Response, Error=HyperError> + Clone + 'static,
S::Future: 'static
where
S: Service<Request = Request, Response = Response, Error = HyperError> + Clone + 'static,
S::Future: 'static,
{
type Request = ();
type Response = Vec<CratePath>;
type Error = Error;
type Future = Box<Future<Item=Self::Response, Error=Self::Error>>;
type Future = Box<dyn Future<Item = Self::Response, Error = Self::Error>>;
fn call(&self, _req: ()) -> Self::Future {
let service = self.0.clone();
let uri_future = format!("{}/summary", CRATES_API_BASE_URI)
.parse::<Uri>().into_future().from_err();
.parse::<Uri>()
.into_future()
.from_err();
Box::new(uri_future.and_then(move |uri| {
let request = Request::new(Method::Get, uri.clone());
@ -151,7 +174,11 @@ impl<S> Service for GetPopularCrates<S>
service.call(request).from_err().and_then(move |response| {
let status = response.status();
if !status.is_success() {
future::Either::A(future::err(format_err!("Status code {} for URI {}", status, uri)))
future::Either::A(future::err(anyhow!(
"Status code {} for URI {}",
status,
uri
)))
} else {
let body_future = response.body().concat2().from_err();
let decode_future = body_future.and_then(|body| {

61
src/interactors/github.rs
View file

@ -1,58 +1,64 @@
use failure::Error;
use anyhow::{anyhow, ensure, Error};
use futures::{Future, Stream};
use hyper::{Error as HyperError, Method, Request, Response, Uri};
use hyper::header::UserAgent;
use hyper::{Error as HyperError, Method, Request, Response, Uri};
use relative_path::RelativePathBuf;
use serde::Deserialize;
use tokio_service::Service;
use serde_json;
use ::models::repo::{Repository, RepoPath};
use crate::models::repo::{RepoPath, Repository};
const GITHUB_API_BASE_URI: &'static str = "https://api.github.com";
const GITHUB_USER_CONTENT_BASE_URI: &'static str = "https://raw.githubusercontent.com";
pub fn get_manifest_uri(repo_path: &RepoPath, path: &RelativePathBuf) -> Result<Uri, Error> {
let path_str: &str = path.as_ref();
Ok(format!("{}/{}/{}/HEAD/{}",
Ok(format!(
"{}/{}/{}/HEAD/{}",
GITHUB_USER_CONTENT_BASE_URI,
repo_path.qual.as_ref(),
repo_path.name.as_ref(),
path_str
).parse::<Uri>()?)
)
.parse::<Uri>()?)
}
#[derive(Deserialize)]
struct GithubSearchResponse {
items: Vec<GithubRepo>
items: Vec<GithubRepo>,
}
#[derive(Deserialize)]
struct GithubRepo {
name: String,
owner: GithubOwner,
description: String
description: String,
}
#[derive(Deserialize)]
struct GithubOwner {
login: String
login: String,
}
#[derive(Debug, Clone)]
pub struct GetPopularRepos<S>(pub S);
impl<S> Service for GetPopularRepos<S>
where S: Service<Request=Request, Response=Response, Error=HyperError> + Clone + 'static,
S::Future: 'static
where
S: Service<Request = Request, Response = Response, Error = HyperError> + Clone + 'static,
S::Future: 'static,
{
type Request = ();
type Response = Vec<Repository>;
type Error = Error;
type Future = Box<Future<Item=Self::Response, Error=Self::Error>>;
type Future = Box<dyn Future<Item = Self::Response, Error = Self::Error>>;
fn call(&self, _req: ()) -> Self::Future {
let uri = try_future_box!(format!("{}/search/repositories?q=language:rust&sort=stars", GITHUB_API_BASE_URI)
.parse::<Uri>());
let uri = try_future_box!(format!(
"{}/search/repositories?q=language:rust&sort=stars",
GITHUB_API_BASE_URI
)
.parse::<Uri>());
let mut request = Request::new(Method::Get, uri);
request.headers_mut().set(UserAgent::new("deps.rs"));
@ -60,18 +66,31 @@ impl<S> Service for GetPopularRepos<S>
Box::new(self.0.call(request).from_err().and_then(|response| {
let status = response.status();
if !status.is_success() {
try_future!(Err(format_err!("Status code {} for popular repo search", status)));
try_future!(Err(anyhow!(
"Status code {} for popular repo search",
status
)));
}
let body_future = response.body().concat2().from_err();
let decode_future = body_future
.and_then(|body| serde_json::from_slice(body.as_ref()).map_err(|err| err.into()));
decode_future.and_then(|search_response: GithubSearchResponse| {
search_response.items.into_iter().map(|item| {
let path = RepoPath::from_parts("github", &item.owner.login, &item.name)?;
Ok(Repository { path, description: item.description })
}).collect::<Result<Vec<_>, _>>()
}).into()
decode_future
.and_then(|search_response: GithubSearchResponse| {
search_response
.items
.into_iter()
.map(|item| {
let path =
RepoPath::from_parts("github", &item.owner.login, &item.name)?;
Ok(Repository {
path,
description: item.description,
})
})
.collect::<Result<Vec<_>, _>>()
})
.into()
}))
}
}

10
src/interactors/gitlab.rs
View file

@ -1,8 +1,8 @@
use anyhow::{anyhow, ensure, Error};
use hyper::Uri;
use relative_path::RelativePathBuf;
use failure::Error;
use ::models::repo::RepoPath;
use crate::models::repo::RepoPath;
const GITLAB_USER_CONTENT_BASE_URI: &'static str = "https://gitlab.com";
@ -15,10 +15,12 @@ pub fn get_manifest_uri(repo_path: &RepoPath, path: &RelativePathBuf) -> Result<
} else {
path_str
};
Ok(format!("{}/{}/{}/raw/HEAD/{}",
Ok(format!(
"{}/{}/{}/raw/HEAD/{}",
GITLAB_USER_CONTENT_BASE_URI,
repo_path.qual.as_ref(),
repo_path.name.as_ref(),
slash_path
).parse::<Uri>()?)
)
.parse::<Uri>()?)
}

27
src/interactors/mod.rs
View file

@ -1,10 +1,10 @@
use failure::Error;
use anyhow::{anyhow, ensure, Error};
use futures::{Future, Stream};
use hyper::{Error as HyperError, Method, Request, Response};
use relative_path::RelativePathBuf;
use tokio_service::Service;
use ::models::repo::{RepoSite, RepoPath};
use crate::models::repo::{RepoPath, RepoSite};
pub mod bitbucket;
pub mod crates;
@ -16,26 +16,21 @@ pub mod rustsec;
pub struct RetrieveFileAtPath<S>(pub S);
impl<S> Service for RetrieveFileAtPath<S>
where S: Service<Request=Request, Response=Response, Error=HyperError> + Clone + 'static,
S::Future: 'static
where
S: Service<Request = Request, Response = Response, Error = HyperError> + Clone + 'static,
S::Future: 'static,
{
type Request = (RepoPath, RelativePathBuf);
type Response = String;
type Error = Error;
type Future = Box<Future<Item=Self::Response, Error=Self::Error>>;
type Future = Box<dyn Future<Item = Self::Response, Error = Self::Error>>;
fn call(&self, req: Self::Request) -> Self::Future {
let (repo_path, path) = req;
let uri = match &repo_path.site {
&RepoSite::Github => {
try_future_box!(github::get_manifest_uri(&repo_path, &path))
},
&RepoSite::Gitlab => {
try_future_box!(gitlab::get_manifest_uri(&repo_path, &path))
},
&RepoSite::Bitbucket => {
try_future_box!(bitbucket::get_manifest_uri(&repo_path, &path))
}
&RepoSite::Github => try_future_box!(github::get_manifest_uri(&repo_path, &path)),
&RepoSite::Gitlab => try_future_box!(gitlab::get_manifest_uri(&repo_path, &path)),
&RepoSite::Bitbucket => try_future_box!(bitbucket::get_manifest_uri(&repo_path, &path)),
};
let request = Request::new(Method::Get, uri.clone());
@ -43,7 +38,7 @@ impl<S> Service for RetrieveFileAtPath<S>
Box::new(self.0.call(request).from_err().and_then(move |response| {
let status = response.status();
if !status.is_success() {
try_future!(Err(format_err!("Status code {} for URI {}", status, uri)));
try_future!(Err(anyhow!("Status code {} for URI {}", status, uri)));
}
let body_future = response.body().concat2().from_err();
@ -54,5 +49,3 @@ impl<S> Service for RetrieveFileAtPath<S>
}))
}
}

81
src/interactors/rustsec.rs
View file

@ -1,44 +1,75 @@
use std::str;
use std::sync::Arc;
use failure::Error;
use futures::{Future, IntoFuture, Stream, future};
use anyhow::{anyhow, ensure, Error};
use futures::{future, future::done, Future, IntoFuture, Stream};
use hyper::{Error as HyperError, Method, Request, Response};
use rustsec::ADVISORY_DB_URL;
use rustsec::db::AdvisoryDatabase;
use rustsec::database::Database;
use rustsec::repository::DEFAULT_URL;
use tokio_service::Service;
#[derive(Debug, Clone)]
pub struct FetchAdvisoryDatabase<S>(pub S);
impl<S> Service for FetchAdvisoryDatabase<S>
where S: Service<Request=Request, Response=Response, Error=HyperError> + Clone + 'static,
S::Future: 'static
where
S: Service<Request = Request, Response = Response, Error = HyperError> + Clone + 'static,
S::Future: 'static,
{
type Request = ();
type Response = Arc<AdvisoryDatabase>;
type Response = Arc<Database>;
type Error = Error;
type Future = Box<Future<Item=Self::Response, Error=Self::Error>>;
type Future = Box<dyn Future<Item = Self::Response, Error = Self::Error>>;
fn call(&self, _req: ()) -> Self::Future {
let service = self.0.clone();
let uri_future = ADVISORY_DB_URL.parse().into_future().from_err();
Box::new(uri_future.and_then(move |uri| {
let request = Request::new(Method::Get, uri);
service.call(request).from_err().and_then(|response| {
let status = response.status();
if !status.is_success() {
future::Either::A(future::err(format_err!("Status code {} when fetching advisory db", status)))
} else {
let body_future = response.body().concat2().from_err();
let decode_future = body_future
.and_then(|body| Ok(Arc::new(AdvisoryDatabase::from_toml(str::from_utf8(&body)?)?)));
future::Either::B(decode_future)
}
})
}))
Box::new(done(
rustsec::Database::fetch()
.map(|db| Arc::new(db))
.map_err(|err| anyhow!("err fetching rustsec DB")),
))
}
}
// #[derive(Debug, Clone)]
// pub struct FetchAdvisoryDatabase<S>(pub S);
// impl<S> Service for FetchAdvisoryDatabase<S>
// where
// S: Service<Request = Request, Response = Response, Error = HyperError> + Clone + 'static,
// S::Future: 'static,
// {
// type Request = ();
// type Response = Arc<Database>;
// type Error = Error;
// type Future = Box<dyn Future<Item = Self::Response, Error = Self::Error>>;
// fn call(&self, _req: ()) -> Self::Future {
// let service = self.0.clone();
// let uri_future = DEFAULT_URL.parse().into_future().from_err();
// Box::new(uri_future.and_then(move |uri| {
// let request = Request::new(Method::Get, uri);
// service.call(request).from_err().and_then(|response| {
// let status = response.status();
// if !status.is_success() {
// future::Either::A(future::err(anyhow!(
// "Status code {} when fetching advisory db",
// status
// )))
// } else {
// let body_future = response.body().concat2().from_err();
// let decode_future = body_future.and_then(|body| {
// Ok(Arc::new(Database::from_toml(str::from_utf8(
// &body,
// )?)?))
// });
// future::Either::B(decode_future)
// }
// })
// }))
// }
// }

69
src/main.rs
View file

@ -1,54 +1,32 @@
#![feature(ascii_ctype)]
#![feature(conservative_impl_trait)]
#![feature(ip_constructors)]
#![feature(proc_macro)]
#![deny(rust_2018_idioms)]
#![allow(unused)]
extern crate badge;
extern crate cadence;
#[macro_use] extern crate failure;
#[macro_use] extern crate futures;
extern crate hyper;
extern crate hyper_tls;
extern crate indexmap;
#[macro_use] extern crate lazy_static;
extern crate lru_cache;
extern crate maud;
extern crate relative_path;
extern crate route_recognizer;
extern crate rustsec;
extern crate semver;
#[macro_use] extern crate serde_derive;
extern crate serde;
extern crate serde_json;
extern crate shared_failure;
#[macro_use] extern crate slog;
extern crate slog_json;
extern crate tokio_core;
extern crate tokio_service;
extern crate toml;
#[macro_use] extern crate try_future;
mod utils;
mod models;
mod parsers;
mod interactors;
mod engine;
mod server;
#[macro_use]
extern crate try_future;
use std::env;
use std::net::{IpAddr, Ipv4Addr, UdpSocket, SocketAddr};
use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
use std::sync::Mutex;
use cadence::{QueuingMetricSink, UdpMetricSink};
use futures::{Future, Stream};
use hyper::Client;
use hyper::server::Http;
use hyper::Client;
use hyper_tls::HttpsConnector;
use slog::Drain;
use slog::{info, o};
use tokio_core::reactor::Core;
use self::server::Server;
mod engine;
mod interactors;
mod models;
mod parsers;
mod server;
mod utils;
use self::engine::Engine;
use self::server::Server;
fn init_metrics() -> QueuingMetricSink {
let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
@ -61,27 +39,27 @@ fn init_metrics() -> QueuingMetricSink {
fn main() {
let logger = slog::Logger::root(
Mutex::new(slog_json::Json::default(std::io::stderr())).map(slog::Fuse),
o!("version" => env!("CARGO_PKG_VERSION"))
o!("version" => env!("CARGO_PKG_VERSION")),
);
let metrics = init_metrics();
let mut core = Core::new()
.expect("failed to create event loop");
let mut core = Core::new().expect("failed to create event loop");
let handle = core.handle();
let connector = HttpsConnector::new(4, &handle)
.expect("failed to create https connector");
let connector = HttpsConnector::new(4, &handle).expect("failed to create https connector");
let client = Client::configure()
.connector(connector)
.build(&core.handle());
let port = env::var("PORT").unwrap_or_else(|_| "8080".to_string()).parse()
let port = env::var("PORT")
.unwrap_or_else(|_| "8080".to_string())
.parse()
.expect("could not read port");
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::unspecified()), port);
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port);
let http = Http::new();
@ -90,7 +68,8 @@ fn main() {
let server = Server::new(logger.clone(), engine);
let serve = http.serve_addr_handle(&addr, &handle, move || Ok(server.clone()))
let serve = http
.serve_addr_handle(&addr, &handle, move || Ok(server.clone()))
.expect("failed to bind server");
let serving = serve.for_each(move |conn| {

127
src/models/crates.rs
View file

@ -1,7 +1,7 @@
use std::borrow::Borrow;
use std::str::FromStr;
use failure::Error;
use anyhow::{anyhow, ensure, Error};
use indexmap::IndexMap;
use relative_path::RelativePathBuf;
use semver::{Version, VersionReq};
@ -9,14 +9,14 @@ use semver::{Version, VersionReq};
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct CratePath {
pub name: CrateName,
pub version: Version
pub version: Version,
}
impl CratePath {
pub fn from_parts(name: &str, version: &str) -> Result<CratePath, Error> {
Ok(CratePath {
name: name.parse()?,
version: version.parse()?
version: version.parse()?,
})
}
}
@ -46,12 +46,12 @@ impl FromStr for CrateName {
type Err = Error;
fn from_str(input: &str) -> Result<CrateName, Error> {
let is_valid = input.chars().all(|c| {
c.is_ascii_alphanumeric() || c == '_' || c == '-'
});
let is_valid = input
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-');
if !is_valid {
Err(format_err!("failed to validate crate name: {}", input))
Err(anyhow!("failed to validate crate name: {}", input))
} else {
Ok(CrateName(input.to_string()))
}
@ -63,13 +63,13 @@ pub struct CrateRelease {
pub name: CrateName,
pub version: Version,
pub deps: CrateDeps,
pub yanked: bool
pub yanked: bool,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum CrateDep {
External(VersionReq),
Internal(RelativePathBuf)
Internal(RelativePathBuf),
}
impl CrateDep {
@ -86,7 +86,7 @@ impl CrateDep {
pub struct CrateDeps {
pub main: IndexMap<CrateName, CrateDep>,
pub dev: IndexMap<CrateName, CrateDep>,
pub build: IndexMap<CrateName, CrateDep>
pub build: IndexMap<CrateName, CrateDep>,
}
#[derive(Debug)]
@ -94,7 +94,7 @@ pub struct AnalyzedDependency {
pub required: VersionReq,
pub latest_that_matches: Option<Version>,
pub latest: Option<Version>,
pub insecure: bool
pub insecure: bool,
}
impl AnalyzedDependency {
@ -103,7 +103,7 @@ impl AnalyzedDependency {
required,
latest_that_matches: None,
latest: None,
insecure: false
insecure: false,
}
}
@ -116,32 +116,44 @@ impl AnalyzedDependency {
pub struct AnalyzedDependencies {
pub main: IndexMap<CrateName, AnalyzedDependency>,
pub dev: IndexMap<CrateName, AnalyzedDependency>,
pub build: IndexMap<CrateName, AnalyzedDependency>
pub build: IndexMap<CrateName, AnalyzedDependency>,
}
impl AnalyzedDependencies {
pub fn new(deps: &CrateDeps) -> AnalyzedDependencies {
let main = deps.main.iter().filter_map(|(name, dep)| {
if let &CrateDep::External(ref req) = dep {
Some((name.clone(), AnalyzedDependency::new(req.clone())))
} else {
None
}
}).collect();
let dev = deps.dev.iter().filter_map(|(name, dep)| {
if let &CrateDep::External(ref req) = dep {
Some((name.clone(), AnalyzedDependency::new(req.clone())))
} else {
None
}
}).collect();
let build = deps.build.iter().filter_map(|(name, dep)| {
if let &CrateDep::External(ref req) = dep {
Some((name.clone(), AnalyzedDependency::new(req.clone())))
} else {
None
}
}).collect();
let main = deps
.main
.iter()
.filter_map(|(name, dep)| {
if let &CrateDep::External(ref req) = dep {
Some((name.clone(), AnalyzedDependency::new(req.clone())))
} else {
None
}
})
.collect();
let dev = deps
.dev
.iter()
.filter_map(|(name, dep)| {
if let &CrateDep::External(ref req) = dep {
Some((name.clone(), AnalyzedDependency::new(req.clone())))
} else {
None
}
})
.collect();
let build = deps
.build
.iter()
.filter_map(|(name, dep)| {
if let &CrateDep::External(ref req) = dep {
Some((name.clone(), AnalyzedDependency::new(req.clone())))
} else {
None
}
})
.collect();
AnalyzedDependencies { main, dev, build }
}
@ -150,38 +162,35 @@ impl AnalyzedDependencies {
}
pub fn count_outdated(&self) -> usize {
let main_outdated = self.main.iter()
let main_outdated = self
.main
.iter()
.filter(|&(_, dep)| dep.is_outdated())
.count();
let dev_outdated = self.dev.iter()
let dev_outdated = self
.dev
.iter()
.filter(|&(_, dep)| dep.is_outdated())
.count();
let build_outdated = self.build.iter()
let build_outdated = self
.build
.iter()
.filter(|&(_, dep)| dep.is_outdated())
.count();
main_outdated + dev_outdated + build_outdated
}
pub fn count_insecure(&self) -> usize {
let main_insecure = self.main.iter()
.filter(|&(_, dep)| dep.insecure)
.count();
let dev_insecure = self.dev.iter()
.filter(|&(_, dep)| dep.insecure)
.count();
let build_insecure = self.build.iter()
.filter(|&(_, dep)| dep.insecure)
.count();
pub fn count_insecure(&self) -> usize {
let main_insecure = self.main.iter().filter(|&(_, dep)| dep.insecure).count();
let dev_insecure = self.dev.iter().filter(|&(_, dep)| dep.insecure).count();
let build_insecure = self.build.iter().filter(|&(_, dep)| dep.insecure).count();
main_insecure + dev_insecure + build_insecure
}
}
pub fn any_outdated(&self) -> bool {
let main_any_outdated = self.main.iter()
.any(|(_, dep)| dep.is_outdated());
let dev_any_outdated = self.dev.iter()
.any(|(_, dep)| dep.is_outdated());
let build_any_outdated = self.build.iter()
.any(|(_, dep)| dep.is_outdated());
let main_any_outdated = self.main.iter().any(|(_, dep)| dep.is_outdated());
let dev_any_outdated = self.dev.iter().any(|(_, dep)| dep.is_outdated());
let build_any_outdated = self.build.iter().any(|(_, dep)| dep.is_outdated());
main_any_outdated || dev_any_outdated || build_any_outdated
}
}
@ -189,6 +198,12 @@ impl AnalyzedDependencies {
#[derive(Clone, Debug)]
pub enum CrateManifest {
Package(CrateName, CrateDeps),
Workspace { members: Vec<RelativePathBuf> },
Mixed { name: CrateName, deps: CrateDeps, members: Vec<RelativePathBuf> }
Workspace {
members: Vec<RelativePathBuf>,
},
Mixed {
name: CrateName,
deps: CrateDeps,
members: Vec<RelativePathBuf>,
},
}

2
src/models/mod.rs
View file

@ -3,5 +3,5 @@ pub mod repo;
pub enum SubjectPath {
Repo(self::repo::RepoPath),
Crate(self::crates::CratePath)
Crate(self::crates::CratePath),
}

22
src/models/repo.rs
View file

@ -1,18 +1,18 @@
use std::str::FromStr;
use failure::Error;
use anyhow::{anyhow, ensure, Error};
#[derive(Clone, Debug)]
pub struct Repository {
pub path: RepoPath,
pub description: String
pub description: String,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct RepoPath {
pub site: RepoSite,
pub qual: RepoQualifier,
pub name: RepoName
pub name: RepoName,
}
impl RepoPath {
@ -20,7 +20,7 @@ impl RepoPath {
Ok(RepoPath {
site: site.parse()?,
qual: qual.parse()?,
name: name.parse()?
name: name.parse()?,
})
}
}
@ -50,7 +50,7 @@ impl FromStr for RepoSite {
"github" => Ok(RepoSite::Github),
"gitlab" => Ok(RepoSite::Gitlab),
"bitbucket" => Ok(RepoSite::Bitbucket),
_ => Err(format_err!("unknown repo site identifier"))
_ => Err(anyhow!("unknown repo site identifier")),
}
}
}
@ -72,9 +72,9 @@ impl FromStr for RepoQualifier {
type Err = Error;
fn from_str(input: &str) -> Result<RepoQualifier, Error> {
let is_valid = input.chars().all(|c| {
c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == '_'
});
let is_valid = input
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == '_');
ensure!(is_valid, "invalid repo qualifier");
Ok(RepoQualifier(input.to_string()))
@ -94,9 +94,9 @@ impl FromStr for RepoName {
type Err = Error;
fn from_str(input: &str) -> Result<RepoName, Error> {
let is_valid = input.chars().all(|c| {
c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == '_'
});
let is_valid = input
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == '_');
ensure!(is_valid, "invalid repo name");
Ok(RepoName(input.to_string()))

110
src/parsers/manifest.rs
View file

@ -1,34 +1,34 @@
use failure::Error;
use anyhow::{anyhow, ensure, Error};
use indexmap::IndexMap;
use relative_path::RelativePathBuf;
use semver::VersionReq;
use toml;
use serde::{Deserialize, Serialize};
use ::models::crates::{CrateName, CrateDep, CrateDeps, CrateManifest};
use crate::models::crates::{CrateDep, CrateDeps, CrateManifest, CrateName};
#[derive(Serialize, Deserialize, Debug)]
struct CargoTomlComplexDependency {
git: Option<String>,
path: Option<RelativePathBuf>,
version: Option<String>
version: Option<String>,
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(untagged)]
enum CargoTomlDependency {
Simple(String),
Complex(CargoTomlComplexDependency)
Complex(CargoTomlComplexDependency),
}
#[derive(Serialize, Deserialize, Debug)]
struct CargoTomlPackage {
name: String
name: String,
}
#[derive(Serialize, Deserialize, Debug)]
struct CargoTomlWorkspace {
#[serde(default)]
members: Vec<RelativePathBuf>
members: Vec<RelativePathBuf>,
}
#[derive(Serialize, Deserialize, Debug)]
@ -44,32 +44,42 @@ struct CargoToml {
dev_dependencies: IndexMap<String, CargoTomlDependency>,
#[serde(rename = "build-dependencies")]
#[serde(default)]
build_dependencies: IndexMap<String, CargoTomlDependency>
build_dependencies: IndexMap<String, CargoTomlDependency>,
}
fn convert_dependency(cargo_dep: (String, CargoTomlDependency)) -> Option<Result<(CrateName, CrateDep), Error>> {
fn convert_dependency(
cargo_dep: (String, CargoTomlDependency),
) -> Option<Result<(CrateName, CrateDep), Error>> {
match cargo_dep {
(name, CargoTomlDependency::Simple(string)) => {
Some(name.parse::<CrateName>().map_err(|err| err.into()).and_then(|parsed_name| {
string.parse::<VersionReq>().map_err(|err| err.into())
.map(|version| (parsed_name, CrateDep::External(version)))
}))
}
(name, CargoTomlDependency::Simple(string)) => Some(
name.parse::<CrateName>()
.map_err(|err| err.into())
.and_then(|parsed_name| {
string
.parse::<VersionReq>()
.map_err(|err| err.into())
.map(|version| (parsed_name, CrateDep::External(version)))
}),
),
(name, CargoTomlDependency::Complex(cplx)) => {
if cplx.git.is_some() {
None
} else if cplx.path.is_some() {
cplx.path.map(|path| {
name.parse::<CrateName>().map_err(|err| err.into()).map(|parsed_name| {
(parsed_name, CrateDep::Internal(path))
})
name.parse::<CrateName>()
.map_err(|err| err.into())
.map(|parsed_name| (parsed_name, CrateDep::Internal(path)))
})
} else {
cplx.version.map(|string| {
name.parse::<CrateName>().map_err(|err| err.into()).and_then(|parsed_name| {
string.parse::<VersionReq>().map_err(|err| err.into())
.map(|version| (parsed_name, CrateDep::External(version)))
})
name.parse::<CrateName>()
.map_err(|err| err.into())
.and_then(|parsed_name| {
string
.parse::<VersionReq>()
.map_err(|err| err.into())
.map(|version| (parsed_name, CrateDep::External(version)))
})
})
}
}
@ -82,20 +92,29 @@ pub fn parse_manifest_toml(input: &str) -> Result<CrateManifest, Error> {
let mut package_part = None;
let mut workspace_part = None;
if let Some(package) = cargo_toml.package {
if let Some(package) = cargo_toml.package {
let crate_name = package.name.parse::<CrateName>()?;
let dependencies = cargo_toml.dependencies
.into_iter().filter_map(convert_dependency).collect::<Result<IndexMap<_, _>, _>>()?;
let dev_dependencies = cargo_toml.dev_dependencies
.into_iter().filter_map(convert_dependency).collect::<Result<IndexMap<_, _>, _>>()?;
let build_dependencies = cargo_toml.build_dependencies
.into_iter().filter_map(convert_dependency).collect::<Result<IndexMap<_, _>, _>>()?;
let dependencies = cargo_toml
.dependencies
.into_iter()
.filter_map(convert_dependency)
.collect::<Result<IndexMap<_, _>, _>>()?;
let dev_dependencies = cargo_toml
.dev_dependencies
.into_iter()
.filter_map(convert_dependency)
.collect::<Result<IndexMap<_, _>, _>>()?;
let build_dependencies = cargo_toml
.build_dependencies
.into_iter()
.filter_map(convert_dependency)
.collect::<Result<IndexMap<_, _>, _>>()?;
let deps = CrateDeps {
main: dependencies,
dev: dev_dependencies,
build: build_dependencies
build: build_dependencies,
};
package_part = Some((crate_name, deps));
@ -106,21 +125,22 @@ pub fn parse_manifest_toml(input: &str) -> Result<CrateManifest, Error> {
}
match (package_part, workspace_part) {
(Some((name, deps)), None) =>
Ok(CrateManifest::Package(name, deps)),
(None, Some(members)) =>
Ok(CrateManifest::Workspace { members }),
(Some((name, deps)), Some(members)) =>
Ok(CrateManifest::Mixed { name, deps, members }),
(None, None) =>
Err(format_err!("neither workspace nor package found in manifest"))
(Some((name, deps)), None) => Ok(CrateManifest::Package(name, deps)),
(None, Some(members)) => Ok(CrateManifest::Workspace { members }),
(Some((name, deps)), Some(members)) => Ok(CrateManifest::Mixed {
name,
deps,
members,
}),
(None, None) => Err(anyhow!("neither workspace nor package found in manifest")),
}
}
#[cfg(test)]
mod tests {
use models::crates::CrateManifest;
use super::parse_manifest_toml;
use crate::models::crates::CrateManifest;
use super::*;
#[test]
fn parse_workspace_without_members_declaration() {
@ -136,14 +156,18 @@ symbolic-common = { version = "2.0.6", path = "common" }
let manifest = parse_manifest_toml(toml).unwrap();
match manifest {
CrateManifest::Mixed { name, deps, members } => {
CrateManifest::Mixed {
name,
deps,
members,
} => {
assert_eq!(name.as_ref(), "symbolic");
assert_eq!(deps.main.len(), 1);
assert_eq!(deps.dev.len(), 0);
assert_eq!(deps.build.len(), 0);
assert_eq!(members.len(), 0);
},
_ => panic!("expected mixed manifest")
}
_ => panic!("expected mixed manifest"),
}
}
}

6
src/server/assets.rs
View file

@ -1,4 +1,2 @@
pub static STATIC_STYLE_CSS: &'static str =
include_str!(concat!(env!("OUT_DIR"), "/style.css"));
pub static STATIC_FAVICON_PNG: &'static [u8; 1338] =
include_bytes!("../../assets/favicon.png");
pub static STATIC_STYLE_CSS: &'static str = include_str!(concat!(env!("OUT_DIR"), "/style.css"));
pub static STATIC_FAVICON_PNG: &'static [u8; 1338] = include_bytes!("../../assets/favicon.png");

315
src/server/mod.rs
View file

@ -1,32 +1,34 @@
use std::env;
use std::sync::Arc;
use futures::{Future, IntoFuture, future};
use hyper::{Error as HyperError, Method, Request, Response, StatusCode};
use futures::{future, Future, IntoFuture};
use hyper::header::{ContentType, Location};
use hyper::{Error as HyperError, Method, Request, Response, StatusCode};
use once_cell::sync::Lazy;
use route_recognizer::{Params, Router};
use semver::VersionReq;
use slog::Logger;
use slog::{error, o};
use tokio_service::Service;
mod assets;
mod views;
use ::engine::{Engine, AnalyzeDependenciesOutcome};
use ::models::crates::{CrateName, CratePath};
use ::models::repo::RepoPath;
use ::models::SubjectPath;
use crate::engine::{AnalyzeDependenciesOutcome, Engine};
use crate::models::crates::{CrateName, CratePath};
use crate::models::repo::RepoPath;
use crate::models::SubjectPath;
#[derive(Clone, Copy, PartialEq)]
enum StatusFormat {
Html,
Svg
Svg,
}
#[derive(Clone, Copy)]
enum StaticFile {
StyleCss,
FaviconPng
FaviconPng,
}
enum Route {
@ -34,14 +36,14 @@ enum Route {
Static(StaticFile),
RepoStatus(StatusFormat),
CrateRedirect,
CrateStatus(StatusFormat)
CrateStatus(StatusFormat),
}
#[derive(Clone)]
pub struct Server {
logger: Logger,
engine: Engine,
router: Arc<Router<Route>>
router: Arc<Router<Route>>,
}
impl Server {
@ -53,14 +55,30 @@ impl Server {
router.add("/static/style.css", Route::Static(StaticFile::StyleCss));
router.add("/static/favicon.png", Route::Static(StaticFile::FaviconPng));
router.add("/repo/:site/:qual/:name", Route::RepoStatus(StatusFormat::Html));
router.add("/repo/:site/:qual/:name/status.svg", Route::RepoStatus(StatusFormat::Svg));
router.add(
"/repo/:site/:qual/:name",
Route::RepoStatus(StatusFormat::Html),
);
router.add(
"/repo/:site/:qual/:name/status.svg",
Route::RepoStatus(StatusFormat::Svg),
);
router.add("/crate/:name", Route::CrateRedirect);
router.add("/crate/:name/:version", Route::CrateStatus(StatusFormat::Html));
router.add("/crate/:name/:version/status.svg", Route::CrateStatus(StatusFormat::Svg));
router.add(
"/crate/:name/:version",
Route::CrateStatus(StatusFormat::Html),
);
router.add(
"/crate/:name/:version/status.svg",
Route::CrateStatus(StatusFormat::Svg),
);
Server { logger, engine, router: Arc::new(router) }
Server {
logger,
engine,
router: Arc::new(router),
}
}
}
@ -68,10 +86,12 @@ impl Service for Server {
type Request = Request;
type Response = Response;
type Error = HyperError;
type Future = Box<Future<Item=Response, Error=HyperError>>;
type Future = Box<dyn Future<Item = Response, Error = HyperError>>;
fn call(&self, req: Request) -> Self::Future {
let logger = self.logger.new(o!("http_path" => req.uri().path().to_owned()));
let logger = self
.logger
.new(o!("http_path" => req.uri().path().to_owned()));
if let Ok(route_match) = self.router.recognize(req.uri().path()) {
match route_match.handler {
@ -79,28 +99,32 @@ impl Service for Server {
if *req.method() == Method::Get {
return Box::new(self.index(req, route_match.params, logger));
}
},
}
&Route::RepoStatus(format) => {
if *req.method() == Method::Get {
return Box::new(self.repo_status(req, route_match.params, logger, format));
}
},
}
&Route::CrateStatus(format) => {
if *req.method() == Method::Get {
return Box::new(self.crate_status(req, route_match.params, logger, format));
return Box::new(self.crate_status(
req,
route_match.params,
logger,
format,
));
}
},
}
&Route::CrateRedirect => {
if *req.method() == Method::Get {
return Box::new(self.crate_redirect(req, route_match.params, logger));
}
},
}
&Route::Static(file) => {
if *req.method() == Method::Get {
return Box::new(future::ok(Server::static_file(file)));
}
}
}
}
@ -111,175 +135,216 @@ impl Service for Server {
}
impl Server {
fn index(&self, _req: Request, _params: Params, logger: Logger) ->
impl Future<Item=Response, Error=HyperError>
{
self.engine.get_popular_repos()
fn index(
&self,
_req: Request,
_params: Params,
logger: Logger,
) -> impl Future<Item = Response, Error = HyperError> {
self.engine
.get_popular_repos()
.join(self.engine.get_popular_crates())
.then(move |popular_result| {
match popular_result {
Err(err) => {
error!(logger, "error: {}", err);
let mut response = views::html::error::render("Could not retrieve popular items", "");
response.set_status(StatusCode::InternalServerError);
future::ok(response)
},
Ok((popular_repos, popular_crates)) =>
future::ok(views::html::index::render(popular_repos, popular_crates))
.then(move |popular_result| match popular_result {
Err(err) => {
error!(logger, "error: {}", err);
let mut response =
views::html::error::render("Could not retrieve popular items", "");
response.set_status(StatusCode::InternalServerError);
future::ok(response)
}
Ok((popular_repos, popular_crates)) => {
future::ok(views::html::index::render(popular_repos, popular_crates))
}
})
}
fn repo_status(&self, _req: Request, params: Params, logger: Logger, format: StatusFormat) ->
impl Future<Item=Response, Error=HyperError>
{
fn repo_status(
&self,
_req: Request,
params: Params,
logger: Logger,
format: StatusFormat,
) -> impl Future<Item = Response, Error = HyperError> {
let server = self.clone();
let site = params.find("site").expect("route param 'site' not found");
let qual = params.find("qual").expect("route param 'qual' not found");
let name = params.find("name").expect("route param 'name' not found");
RepoPath::from_parts(site, qual, name).into_future().then(move |repo_path_result| {
match repo_path_result {
RepoPath::from_parts(site, qual, name)
.into_future()
.then(move |repo_path_result| match repo_path_result {
Err(err) => {
error!(logger, "error: {}", err);
let mut response = views::html::error::render("Could not parse repository path",
"Please make sure to provide a valid repository path.");
let mut response = views::html::error::render(
"Could not parse repository path",
"Please make sure to provide a valid repository path.",
);
response.set_status(StatusCode::BadRequest);
future::Either::A(future::ok(response))
},
Ok(repo_path) => {
future::Either::B(server.engine.analyze_repo_dependencies(repo_path.clone()).then(move |analyze_result| {
match analyze_result {
}
Ok(repo_path) => future::Either::B(
server
.engine
.analyze_repo_dependencies(repo_path.clone())
.then(move |analyze_result| match analyze_result {
Err(err) => {
error!(logger, "error: {}", err);
let response = Server::status_format_analysis(None, format, SubjectPath::Repo(repo_path));
future::ok(response)
},
Ok(analysis_outcome) => {
let response = Server::status_format_analysis(Some(analysis_outcome), format, SubjectPath::Repo(repo_path));
let response = Server::status_format_analysis(
None,
format,
SubjectPath::Repo(repo_path),
);
future::ok(response)
}
}
}))
}
}
})
Ok(analysis_outcome) => {
let response = Server::status_format_analysis(
Some(analysis_outcome),
format,
SubjectPath::Repo(repo_path),
);
future::ok(response)
}
}),
),
})
}
fn crate_redirect(&self, _req: Request, params: Params, logger: Logger) ->
impl Future<Item=Response, Error=HyperError>
{
fn crate_redirect(
&self,
_req: Request,
params: Params,
logger: Logger,
) -> impl Future<Item = Response, Error = HyperError> {
let engine = self.engine.clone();
let name = params.find("name").expect("route param 'name' not found");
name.parse::<CrateName>().into_future().then(move |crate_name_result| {
match crate_name_result {
name.parse::<CrateName>()
.into_future()
.then(move |crate_name_result| match crate_name_result {
Err(err) => {
error!(logger, "error: {}", err);
let mut response = views::html::error::render("Could not parse crate name",
"Please make sure to provide a valid crate name.");
let mut response = views::html::error::render(
"Could not parse crate name",
"Please make sure to provide a valid crate name.",
);
response.set_status(StatusCode::BadRequest);
future::Either::A(future::ok(response))
},
Ok(crate_name) => {
future::Either::B(engine.find_latest_crate_release(crate_name, VersionReq::any()).then(move |release_result| {
match release_result {
}
Ok(crate_name) => future::Either::B(
engine
.find_latest_crate_release(crate_name, VersionReq::any())
.then(move |release_result| match release_result {
Err(err) => {
error!(logger, "error: {}", err);
let mut response = views::html::error::render("Could not fetch crate information",
"Please make sure to provide a valid crate name.");
let mut response = views::html::error::render(
"Could not fetch crate information",
"Please make sure to provide a valid crate name.",
);
response.set_status(StatusCode::NotFound);
future::ok(response)
},
}
Ok(None) => {
let mut response = views::html::error::render("Could not fetch crate information",
"Please make sure to provide a valid crate name.");
let mut response = views::html::error::render(
"Could not fetch crate information",
"Please make sure to provide a valid crate name.",
);
response.set_status(StatusCode::NotFound);
future::ok(response)
},
}
Ok(Some(release)) => {
let mut response = Response::new();
response.set_status(StatusCode::TemporaryRedirect);
let url = format!("{}/crate/{}/{}",
let url = format!(
"{}/crate/{}/{}",
&SELF_BASE_URL as &str,
release.name.as_ref(),
release.version);
release.version
);
response.headers_mut().set(Location::new(url));
future::ok(response)
}
}
}))
}
}
})
}),
),
})
}
fn crate_status(&self, _req: Request, params: Params, logger: Logger, format: StatusFormat) ->
impl Future<Item=Response, Error=HyperError>
{
fn crate_status(
&self,
_req: Request,
params: Params,
logger: Logger,
format: StatusFormat,
) -> impl Future<Item = Response, Error = HyperError> {
let server = self.clone();
let name = params.find("name").expect("route param 'name' not found");
let version = params.find("version").expect("route param 'version' not found");
let version = params
.find("version")
.expect("route param 'version' not found");
CratePath::from_parts(name, version).into_future().then(move |crate_path_result| {
match crate_path_result {
CratePath::from_parts(name, version)
.into_future()
.then(move |crate_path_result| match crate_path_result {
Err(err) => {
error!(logger, "error: {}", err);
let mut response = views::html::error::render("Could not parse crate path",
"Please make sure to provide a valid crate name and version.");
let mut response = views::html::error::render(
"Could not parse crate path",
"Please make sure to provide a valid crate name and version.",
);
response.set_status(StatusCode::BadRequest);
future::Either::A(future::ok(response))
},
Ok(crate_path) => {
future::Either::B(server.engine.analyze_crate_dependencies(crate_path.clone()).then(move |analyze_result| {
match analyze_result {
}
Ok(crate_path) => future::Either::B(
server
.engine
.analyze_crate_dependencies(crate_path.clone())
.then(move |analyze_result| match analyze_result {
Err(err) => {
error!(logger, "error: {}", err);
let response = Server::status_format_analysis(None, format, SubjectPath::Crate(crate_path));
future::ok(response)
},
Ok(analysis_outcome) => {
let response = Server::status_format_analysis(Some(analysis_outcome), format, SubjectPath::Crate(crate_path));
let response = Server::status_format_analysis(
None,
format,
SubjectPath::Crate(crate_path),
);
future::ok(response)
}
}
}))
}
}
})
Ok(analysis_outcome) => {
let response = Server::status_format_analysis(
Some(analysis_outcome),
format,
SubjectPath::Crate(crate_path),
);
future::ok(response)
}
}),
),
})
}
fn status_format_analysis(analysis_outcome: Option<AnalyzeDependenciesOutcome>, format: StatusFormat, subject_path: SubjectPath) -> Response {
fn status_format_analysis(
analysis_outcome: Option<AnalyzeDependenciesOutcome>,
format: StatusFormat,
subject_path: SubjectPath,
) -> Response {
match format {
StatusFormat::Svg =>
views::badge::response(analysis_outcome.as_ref()),
StatusFormat::Html =>
views::html::status::render(analysis_outcome, subject_path)
StatusFormat::Svg => views::badge::response(analysis_outcome.as_ref()),
StatusFormat::Html => views::html::status::render(analysis_outcome, subject_path),
}
}
fn static_file(file: StaticFile) -> Response {
match file {
StaticFile::StyleCss => {
Response::new()
.with_header(ContentType("text/css".parse().unwrap()))
.with_body(assets::STATIC_STYLE_CSS)
},
StaticFile::FaviconPng => {
Response::new()
.with_header(ContentType("image/png".parse().unwrap()))
.with_body(assets::STATIC_FAVICON_PNG.to_vec())
}
StaticFile::StyleCss => Response::new()
.with_header(ContentType("text/css".parse().unwrap()))
.with_body(assets::STATIC_STYLE_CSS),
StaticFile::FaviconPng => Response::new()
.with_header(ContentType("image/png".parse().unwrap()))
.with_body(assets::STATIC_FAVICON_PNG.to_vec()),
}
}
}
lazy_static! {
static ref SELF_BASE_URL: String = {
env::var("BASE_URL")
.unwrap_or_else(|_| "http://localhost:8080".to_string())
};
}
static SELF_BASE_URL: Lazy<String> =
Lazy::new(|| env::var("BASE_URL").unwrap_or_else(|_| "http://localhost:8080".to_string()));

24
src/server/views/badge.rs
View file

@ -1,8 +1,8 @@
use badge::{Badge, BadgeOptions};
use hyper::Response;
use hyper::header::ContentType;
use hyper::Response;
use ::engine::AnalyzeDependenciesOutcome;
use crate::engine::AnalyzeDependenciesOutcome;
pub fn badge(analysis_outcome: Option<&AnalyzeDependenciesOutcome>) -> Badge {
let opts = match analysis_outcome {
@ -11,7 +11,7 @@ pub fn badge(analysis_outcome: Option<&AnalyzeDependenciesOutcome>) -> Badge {
BadgeOptions {
subject: "dependencies".into(),
status: "insecure".into(),
color: "#e05d44".into()
color: "#e05d44".into(),
}
} else {
let (outdated, total) = outcome.outdated_ratio();
@ -20,30 +20,28 @@ pub fn badge(analysis_outcome: Option<&AnalyzeDependenciesOutcome>) -> Badge {
BadgeOptions {
subject: "dependencies".into(),
status: format!("{} of {} outdated", outdated, total),
color: "#dfb317".into()
color: "#dfb317".into(),
}
} else if total > 0 {
BadgeOptions {
subject: "dependencies".into(),
status: "up to date".into(),
color: "#4c1".into()
color: "#4c1".into(),
}
} else {
BadgeOptions {
subject: "dependencies".into(),
status: "none".into(),
color: "#4c1".into()
color: "#4c1".into(),
}
}
}
},
None => {
BadgeOptions {
subject: "dependencies".into(),
status: "unknown".into(),
color: "#9f9f9f".into()
}
}
None => BadgeOptions {
subject: "dependencies".into(),
status: "unknown".into(),
color: "#9f9f9f".into(),
},
};
Badge::new(opts)

29
src/server/views/html/error.rs
View file

@ -1,19 +1,22 @@
use hyper::Response;
use maud::html;
pub fn render(title: &str, descr: &str) -> Response {
super::render_html(title, html! {
section class="hero is-light" {
div class="hero-head" (super::render_navbar())
}
section class="section" {
div class="container" {
div class="notification is-danger" {
p class="title is-3" (title)
p (descr)
pub fn render(title: &str, descr: &str) -> Response {
super::render_html(
title,
html! {
section class="hero is-light" {
div class="hero-head" { (super::render_navbar()) }
}
section class="section" {
div class="container" {
div class="notification is-danger" {
p class="title is-3" { (title) }
p { (descr) }
}
}
}
}
(super::render_footer(None))
})
(super::render_footer(None))
},
)
}

53
src/server/views/html/index.rs
View file

@ -1,20 +1,20 @@
use hyper::Response;
use maud::{Markup, html};
use maud::{html, Markup};
use ::models::repo::Repository;
use ::models::crates::CratePath;
use crate::models::crates::CratePath;
use crate::models::repo::Repository;
fn popular_table(popular_repos: Vec<Repository>, popular_crates: Vec<CratePath>) -> Markup {
html! {
div class="columns" {
div class="column" {
h2 class="title is-3" "Popular Repositories"
h2 class="title is-3" { "Popular Repositories" }
table class="table is-fullwidth is-striped is-hoverable" {
thead {
tr {
th "Repository"
th class="has-text-right" "Status"
th { "Repository" }
th class="has-text-right" { "Status" }
}
}
tbody {
@ -34,13 +34,13 @@ fn popular_table(popular_repos: Vec<Repository>, popular_crates: Vec<CratePath>)
}
}
div class="column" {
h2 class="title is-3" "Popular Crates"
h2 class="title is-3" { "Popular Crates" }
table class="table is-fullwidth is-striped is-hoverable" {
thead {
tr {
th "Crate"
th class="has-text-right" "Status"
th { "Crate" }
th class="has-text-right" { "Status" }
}
}
tbody {
@ -64,23 +64,26 @@ fn popular_table(popular_repos: Vec<Repository>, popular_crates: Vec<CratePath>)
}
pub fn render(popular_repos: Vec<Repository>, popular_crates: Vec<CratePath>) -> Response {
super::render_html("Keep your dependencies up-to-date", html! {
section class="hero is-light" {
div class="hero-head" (super::render_navbar())
div class="hero-body" {
div class="container" {
p class="title is-1" "Keep your dependencies up-to-date"
p {
"Deps.rs uses semantic versioning to detect outdated or insecure dependencies in your project's"
code "Cargo.toml"
"."
super::render_html(
"Keep your dependencies up-to-date",
html! {
section class="hero is-light" {
div class="hero-head" { (super::render_navbar()) }
div class="hero-body" {
div class="container" {
p class="title is-1" { "Keep your dependencies up-to-date" }
p {
"Deps.rs uses semantic versioning to detect outdated or insecure dependencies in your project's"
code { "Cargo.toml" }
"."
}
}
}
}
}
section class="section" {
div class="container" (popular_table(popular_repos, popular_crates))
}
(super::render_footer(None))
})
section class="section" {
div class="container" { (popular_table(popular_repos, popular_crates)) }
}
(super::render_footer(None))
},
)
}

23
src/server/views/html/mod.rs
View file

@ -1,11 +1,11 @@
use std::time::Duration;
use hyper::Response;
use hyper::header::ContentType;
use maud::{Markup, Render, html};
use hyper::Response;
use maud::{html, Markup, Render};
pub mod index;
pub mod error;
pub mod index;
pub mod status;
use super::super::SELF_BASE_URL;
@ -16,14 +16,14 @@ fn render_html<B: Render>(title: &str, body: B) -> Response {
head {
meta charset="utf-8";
meta name="viewport" content="width=device-width, initial-scale=1";
title (format!("{} - Deps.rs", title))
title { (format!("{} - Deps.rs", title)) }
link rel="icon" type="image/png" href="/static/favicon.png";
link rel="stylesheet" type="text/css" href="/static/style.css";
link rel="stylesheet" type="text/css" href="https://fonts.googleapis.com/css?family=Fira+Sans:400,500,600";
link rel="stylesheet" type="text/css" href="https://fonts.googleapis.com/css?family=Source+Code+Pro";
link rel="stylesheet" type="text/css" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css";
}
body (body)
body { (body) }
}
};
@ -38,7 +38,7 @@ fn render_navbar() -> Markup {
div class="container" {
div class="navbar-brand" {
a class="navbar-item is-dark" href=(SELF_BASE_URL) {
h1 class="title is-3" "Deps.rs"
h1 class="title is-3" { "Deps.rs" }
}
}
}
@ -47,25 +47,26 @@ fn render_navbar() -> Markup {
}
fn render_footer(duration: Option<Duration>) -> Markup {
let duration_millis = duration.map(|d| d.as_secs() * 1000 + (d.subsec_nanos() / 1000 / 1000) as u64);
let duration_millis =
duration.map(|d| d.as_secs() * 1000 + (d.subsec_nanos() / 1000 / 1000) as u64);
html! {
footer class="footer" {
div class="container" {
div class="content has-text-centered" {
p {
strong "Deps.rs"
strong { "Deps.rs" }
" is a service for the Rust community. It is open source on "
a href="https://github.com/srijs/deps.rs" "GitHub"
a href="https://github.com/srijs/deps.rs" { "GitHub" }
"."
}
p {
"Please report any issues on the "
a href="https://github.com/srijs/deps.rs/issues" "issue tracker"
a href="https://github.com/srijs/deps.rs/issues" { "issue tracker" }
"."
}
@if let Some(millis) = duration_millis {
p class="has-text-grey is-size-7" (format!("(rendered in {} ms)", millis))
p class="has-text-grey is-size-7" { (format!("(rendered in {} ms)", millis)) }
}
}
}

83
src/server/views/html/status.rs
View file

@ -1,11 +1,11 @@
use hyper::Response;
use maud::{Markup, html};
use indexmap::IndexMap;
use maud::{html, Markup};
use ::engine::AnalyzeDependenciesOutcome;
use ::models::crates::{CrateName, AnalyzedDependency, AnalyzedDependencies};
use ::models::SubjectPath;
use ::models::repo::RepoSite;
use crate::engine::AnalyzeDependenciesOutcome;
use crate::models::crates::{AnalyzedDependencies, AnalyzedDependency, CrateName};
use crate::models::repo::RepoSite;
use crate::models::SubjectPath;
use super::super::badge;
@ -13,11 +13,11 @@ fn dependency_tables(crate_name: CrateName, deps: AnalyzedDependencies) -> Marku
html! {
h2 class="title is-3" {
"Crate "
code (crate_name.as_ref())
code { (crate_name.as_ref()) }
}
@if deps.main.is_empty() && deps.dev.is_empty() && deps.build.is_empty() {
p class="notification has-text-centered" "No external dependencies! 🙌"
p class="notification has-text-centered" { "No external dependencies! 🙌" }
}
@if !deps.main.is_empty() {
@ -40,7 +40,7 @@ fn dependency_table(title: &str, deps: IndexMap<CrateName, AnalyzedDependency>)
let count_outdated = deps.iter().filter(|&(_, dep)| dep.is_outdated()).count();
html! {
h3 class="title is-4" (title)
h3 class="title is-4" { (title) }
p class="subtitle is-5" {
@if count_insecure > 0 {
(format!(" ({} total, {} insecure)", count_total, count_insecure))
@ -54,33 +54,33 @@ fn dependency_table(title: &str, deps: IndexMap<CrateName, AnalyzedDependency>)
table class="table is-fullwidth is-striped is-hoverable" {
thead {
tr {
th "Crate"
th class="has-text-right" "Required"
th class="has-text-right" "Latest"
th class="has-text-right" "Status"
th { "Crate" }
th class="has-text-right" { "Required" }
th class="has-text-right" { "Latest" }
th class="has-text-right" { "Status" }
}
}
tbody {
@for (name, dep) in deps {
tr {
td {
a href=(format!("https://crates.io/crates/{}", name.as_ref())) (name.as_ref())
a href=(format!("https://crates.io/crates/{}", name.as_ref())) { (name.as_ref()) }
}
td class="has-text-right" code (dep.required.to_string())
td class="has-text-right" { code { (dep.required.to_string()) } }
td class="has-text-right" {
@if let Some(ref latest) = dep.latest {
code (latest.to_string())
code { (latest.to_string()) }
} @else {
"N/A"
}
}
td class="has-text-right" {
@if dep.insecure {
span class="tag is-danger" "insecure"
span class="tag is-danger" { "insecure" }
} @else if dep.is_outdated() {
span class="tag is-warning" "out of date"
span class="tag is-warning" { "out of date" }
} @else {
span class="tag is-success" "up to date"
span class="tag is-success" { "up to date" }
}
}
}
@ -94,7 +94,7 @@ fn get_site_icon(site: &RepoSite) -> &'static str {
match *site {
RepoSite::Github => "fa-github",
RepoSite::Gitlab => "fa-gitlab",
RepoSite::Bitbucket => "fa-bitbucket"
RepoSite::Bitbucket => "fa-bitbucket",
}
}
@ -104,15 +104,15 @@ fn render_title(subject_path: &SubjectPath) -> Markup {
let site_icon = get_site_icon(&repo_path.site);
html! {
a href=(format!("{}/{}/{}", repo_path.site.to_base_uri(), repo_path.qual.as_ref(), repo_path.name.as_ref())) {
i class=(format!("fa {}", site_icon)) ""
i class=(format!("fa {}", site_icon)) { "" }
(format!(" {} / {}", repo_path.qual.as_ref(), repo_path.name.as_ref()))
}
}
},
}
SubjectPath::Crate(ref crate_path) => {
html! {
a href=(format!("https://crates.io/crates/{}/{}", crate_path.name.as_ref(), crate_path.version)) {
i class="fa fa-cube" ""
i class="fa fa-cube" { "" }
(format!(" {} {}", crate_path.name.as_ref(), crate_path.version))
}
}
@ -123,7 +123,7 @@ fn render_title(subject_path: &SubjectPath) -> Markup {
fn render_failure(subject_path: SubjectPath) -> Markup {
html! {
section class="hero is-light" {
div class="hero-head" (super::render_navbar())
div class="hero-head" { (super::render_navbar()) }
div class="hero-body" {
div class="container" {
h1 class="title is-1" {
@ -135,8 +135,8 @@ fn render_failure(subject_path: SubjectPath) -> Markup {
section class="section" {
div class="container" {
div class="notification is-danger" {
h2 class="title is-3" "Failed to analyze repository"
p "The repository you requested might be structured in an uncommon way that is not yet supported."
h2 class="title is-3" { "Failed to analyze repository" }
p { "The repository you requested might be structured in an uncommon way that is not yet supported." }
}
}
}
@ -144,18 +144,26 @@ fn render_failure(subject_path: SubjectPath) -> Markup {
}
}
fn render_success(analysis_outcome: AnalyzeDependenciesOutcome, subject_path: SubjectPath) -> Markup {
fn render_success(
analysis_outcome: AnalyzeDependenciesOutcome,
subject_path: SubjectPath,
) -> Markup {
let self_path = match subject_path {
SubjectPath::Repo(ref repo_path) =>
format!("repo/{}/{}/{}", repo_path.site.as_ref(), repo_path.qual.as_ref(), repo_path.name.as_ref()),
SubjectPath::Crate(ref crate_path) =>
SubjectPath::Repo(ref repo_path) => format!(
"repo/{}/{}/{}",
repo_path.site.as_ref(),
repo_path.qual.as_ref(),
repo_path.name.as_ref()
),
SubjectPath::Crate(ref crate_path) => {
format!("crate/{}/{}", crate_path.name.as_ref(), crate_path.version)
}
};
let status_base_url = format!("{}/{}", &super::SELF_BASE_URL as &str, self_path);
let status_data_uri = badge::badge(Some(&analysis_outcome)).to_svg_data_uri();
let hero_class = if analysis_outcome.any_insecure() {
let hero_class = if analysis_outcome.any_insecure() {
"is-danger"
} else if analysis_outcome.any_outdated() {
"is-warning"
@ -165,7 +173,7 @@ fn render_success(analysis_outcome: AnalyzeDependenciesOutcome, subject_path: Su
html! {
section class=(format!("hero {}", hero_class)) {
div class="hero-head" (super::render_navbar())
div class="hero-head" { (super::render_navbar()) }
div class="hero-body" {
div class="container" {
h1 class="title is-1" {
@ -194,12 +202,17 @@ fn render_success(analysis_outcome: AnalyzeDependenciesOutcome, subject_path: Su
}
}
pub fn render(analysis_outcome: Option<AnalyzeDependenciesOutcome>, subject_path: SubjectPath) -> Response {
pub fn render(
analysis_outcome: Option<AnalyzeDependenciesOutcome>,
subject_path: SubjectPath,
) -> Response {
let title = match subject_path {
SubjectPath::Repo(ref repo_path) =>
format!("{} / {}", repo_path.qual.as_ref(), repo_path.name.as_ref()),
SubjectPath::Crate(ref crate_path) =>
SubjectPath::Repo(ref repo_path) => {
format!("{} / {}", repo_path.qual.as_ref(), repo_path.name.as_ref())
}
SubjectPath::Crate(ref crate_path) => {
format!("{} {}", crate_path.name.as_ref(), crate_path.version)
}
};
if let Some(outcome) = analysis_outcome {

2
src/server/views/mod.rs
View file

@ -1,2 +1,2 @@
pub mod html;
pub mod badge;
pub mod html;

57
src/utils/cache.rs
View file

@ -1,30 +1,31 @@
use std::fmt::{Debug, Formatter, Result as FmtResult};
use std::hash::Hash;
use std::time::{Duration, Instant};
use std::ops::Deref;
use std::sync::Mutex;
use std::time::{Duration, Instant};
use failure::Error;
use futures::{Future, Poll};
use anyhow::{anyhow, ensure, Error};
use futures::future::{FromErr, Shared, SharedItem};
use futures::{Future, Poll};
use lru_cache::LruCache;
use shared_failure::SharedFailure;
use tokio_service::Service;
pub struct Cache<S>
where S: Service<Error=Error>,
S::Request: Hash + Eq
where
S: Service<Error = Error>,
S::Request: Hash + Eq,
{
inner: S,
duration: Duration,
cache: Mutex<LruCache<S::Request, (Instant, Shared<FromErr<S::Future, SharedFailure>>)>>
cache: Mutex<LruCache<S::Request, (Instant, Shared<FromErr<S::Future, Error>>)>>,
}
impl<S> Debug for Cache<S>
where S: Service<Error=Error> + Debug,
S::Request: Hash + Eq
where
S: Service<Error = Error> + Debug,
S::Request: Hash + Eq,
{
fn fmt(&self, fmt: &mut Formatter) -> FmtResult {
fn fmt(&self, fmt: &mut Formatter<'_>) -> FmtResult {
fmt.debug_struct("Cache")
.field("inner", &self.inner)
.field("duration", &self.duration)
@ -32,26 +33,28 @@ impl<S> Debug for Cache<S>
}
}
impl<S> Cache<S>
where S: Service<Error=Error>,
S::Request: Hash + Eq
impl<S> Cache<S>
where
S: Service<Error = Error>,
S::Request: Hash + Eq,
{
pub fn new(service: S, duration: Duration, capacity: usize) -> Cache<S> {
Cache {
inner: service,
duration: duration,
cache: Mutex::new(LruCache::new(capacity))
cache: Mutex::new(LruCache::new(capacity)),
}
}
}
impl<S> Service for Cache<S>
where S: Service<Error=Error>,
S::Request: Clone + Hash + Eq
where
S: Service<Error = Error>,
S::Request: Clone + Hash + Eq,
{
type Request = S::Request;
type Response = CachedItem<S::Response>;
type Error = SharedFailure;
type Error = Error;
type Future = Cached<S::Future>;
fn call(&self, req: Self::Request) -> Self::Future {
@ -70,25 +73,27 @@ impl<S> Service for Cache<S>
}
}
pub struct Cached<F: Future<Error=Error>>(Shared<FromErr<F, SharedFailure>>);
pub struct Cached<F: Future<Error = Error>>(Shared<FromErr<F, Error>>);
impl<F> Debug for Cached<F>
where F: Future<Error=Error> + Debug,
F::Item: Debug
where
F: Future<Error = Error> + Debug,
F::Item: Debug,
{
fn fmt(&self, fmt: &mut Formatter) -> FmtResult {
fn fmt(&self, fmt: &mut Formatter<'_>) -> FmtResult {
self.0.fmt(fmt)
}
}
impl<F: Future<Error=Error>> Future for Cached<F> {
impl<F: Future<Error = Error>> Future for Cached<F> {
type Item = CachedItem<F::Item>;
type Error = SharedFailure;
type Error = Error;
fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
self.0.poll()
.map_err(|err| (*err).clone())
.map(|async| async.map(CachedItem))
self.0
.poll()
.map_err(|_err| anyhow!("TODO: shared error not clone-able"))
.map(|item| item.map(CachedItem))
}
}