feliix42/deps.rs
1
0
Fork 0
mirror of https://github.com/deps-rs/deps.rs.git synced 2024-11-22 02:16:30 +00:00
Code Issues Projects Releases Packages Wiki Activity

refactor: migrate to tracing for logging

Browse source
This commit is contained in:
Rob Ede 2024-05-26 21:36:03 +01:00
parent 0153222542
commit 85a077e80d
No known key found for this signature in database
GPG key ID: 97C636207D3EF933
23 changed files with 299 additions and 261 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -4,3 +4,6 @@
# These are backup files generated by rustfmt # These are backup files generated by rustfmt
**/*.rs.bk **/*.rs.bk
# local environment variables
.env

3
.rustfmt.toml Normal file
View file

@ -0,0 +1,3 @@
imports_granularity = "Crate"
group_imports = "StdExternalCrate"
use_field_init_shorthand = true

242
Cargo.lock generated
View file

@ -35,6 +35,15 @@ dependencies = [
"zerocopy", "zerocopy",
] ]
[[package]]
name = "aho-corasick"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "allocator-api2" name = "allocator-api2"
version = "0.2.18" version = "0.2.18"
@ -144,7 +153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706"
dependencies = [ dependencies = [
"memchr", "memchr",
"regex-automata", "regex-automata 0.4.6",
"serde", "serde",
] ]
@ -380,25 +389,10 @@ dependencies = [
] ]
[[package]] [[package]]
name = "dirs-next" name = "dotenvy"
version = "2.0.0" version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
dependencies = [
"cfg-if",
"dirs-sys-next",
]
[[package]]
name = "dirs-sys-next"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]] [[package]]
name = "dunce" name = "dunce"
@ -437,6 +431,12 @@ dependencies = [
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
[[package]]
name = "error_reporter"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31ae425815400e5ed474178a7a22e275a9687086a12ca63ec793ff292d8fdae8"
[[package]] [[package]]
name = "faster-hex" name = "faster-hex"
version = "0.9.0" version = "0.9.0"
@ -1673,17 +1673,6 @@ version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
[[package]]
name = "is-terminal"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.52.0",
]
[[package]] [[package]]
name = "itoa" name = "itoa"
version = "1.0.11" version = "1.0.11"
@ -1739,16 +1728,6 @@ version = "0.2.155"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
[[package]]
name = "libredox"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
dependencies = [
"bitflags 2.5.0",
"libc",
]
[[package]] [[package]]
name = "linux-raw-sys" name = "linux-raw-sys"
version = "0.4.14" version = "0.4.14"
@ -1777,6 +1756,15 @@ version = "0.11.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd"
[[package]]
name = "matchers"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
dependencies = [
"regex-automata 0.1.10",
]
[[package]] [[package]]
name = "maud" name = "maud"
version = "0.26.0" version = "0.26.0"
@ -1869,6 +1857,16 @@ dependencies = [
"tempfile", "tempfile",
] ]
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
dependencies = [
"overload",
"winapi",
]
[[package]] [[package]]
name = "num-conv" name = "num-conv"
version = "0.1.0" version = "0.1.0"
@ -1953,6 +1951,12 @@ dependencies = [
"vcpkg", "vcpkg",
] ]
[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]] [[package]]
name = "owned_ttf_parser" name = "owned_ttf_parser"
version = "0.15.2" version = "0.15.2"
@ -2206,14 +2210,24 @@ dependencies = [
] ]
[[package]] [[package]]
name = "redox_users" name = "regex"
version = "0.4.5" version = "1.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
dependencies = [ dependencies = [
"getrandom", "aho-corasick",
"libredox", "memchr",
"thiserror", "regex-automata 0.4.6",
"regex-syntax 0.8.3",
]
[[package]]
name = "regex-automata"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
dependencies = [
"regex-syntax 0.6.29",
] ]
[[package]] [[package]]
@ -2221,6 +2235,23 @@ name = "regex-automata"
version = "0.4.6" version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax 0.8.3",
]
[[package]]
name = "regex-syntax"
version = "0.6.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]]
name = "regex-syntax"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56"
[[package]] [[package]]
name = "relative-path" name = "relative-path"
@ -2422,12 +2453,6 @@ dependencies = [
"owned_ttf_parser", "owned_ttf_parser",
] ]
[[package]]
name = "rustversion"
version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6"
[[package]] [[package]]
name = "ryu" name = "ryu"
version = "1.0.18" version = "1.0.18"
@ -2559,6 +2584,15 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
[[package]]
name = "sharded-slab"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
dependencies = [
"lazy_static",
]
[[package]] [[package]]
name = "shell-words" name = "shell-words"
version = "1.1.0" version = "1.1.0"
@ -2574,6 +2608,8 @@ dependencies = [
"cadence", "cadence",
"crates-index", "crates-index",
"derive_more", "derive_more",
"dotenvy",
"error_reporter",
"font-awesome-as-a-crate", "font-awesome-as-a-crate",
"futures-util", "futures-util",
"gix", "gix",
@ -2583,6 +2619,7 @@ dependencies = [
"lru_time_cache", "lru_time_cache",
"maud", "maud",
"once_cell", "once_cell",
"parking_lot",
"pulldown-cmark", "pulldown-cmark",
"relative-path", "relative-path",
"reqwest", "reqwest",
@ -2592,11 +2629,10 @@ dependencies = [
"serde", "serde",
"serde_urlencoded", "serde_urlencoded",
"sha-1", "sha-1",
"slog",
"slog-async",
"slog-term",
"tokio", "tokio",
"toml 0.8.13", "toml 0.8.13",
"tracing",
"tracing-subscriber",
] ]
[[package]] [[package]]
@ -2614,37 +2650,6 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "slog"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06"
[[package]]
name = "slog-async"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84"
dependencies = [
"crossbeam-channel",
"slog",
"take_mut",
"thread_local",
]
[[package]]
name = "slog-term"
version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8"
dependencies = [
"is-terminal",
"slog",
"term",
"thread_local",
"time",
]
[[package]] [[package]]
name = "smallvec" name = "smallvec"
version = "1.13.2" version = "1.13.2"
@ -2737,12 +2742,6 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "take_mut"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60"
[[package]] [[package]]
name = "tame-index" name = "tame-index"
version = "0.12.0" version = "0.12.0"
@ -2780,17 +2779,6 @@ dependencies = [
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
[[package]]
name = "term"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
dependencies = [
"dirs-next",
"rustversion",
"winapi",
]
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.61" version = "1.0.61"
@ -3033,9 +3021,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
dependencies = [ dependencies = [
"pin-project-lite", "pin-project-lite",
"tracing-attributes",
"tracing-core", "tracing-core",
] ]
[[package]]
name = "tracing-attributes"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
]
[[package]] [[package]]
name = "tracing-core" name = "tracing-core"
version = "0.1.32" version = "0.1.32"
@ -3043,6 +3043,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
dependencies = [ dependencies = [
"once_cell", "once_cell",
"valuable",
]
[[package]]
name = "tracing-log"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
"log",
"once_cell",
"tracing-core",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
dependencies = [
"matchers",
"nu-ansi-term",
"once_cell",
"regex",
"sharded-slab",
"smallvec",
"thread_local",
"tracing",
"tracing-core",
"tracing-log",
] ]
[[package]] [[package]]
@ -3142,6 +3172,12 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "valuable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]] [[package]]
name = "vcpkg" name = "vcpkg"
version = "0.2.15" version = "0.2.15"

8
Cargo.toml
View file

@ -18,13 +18,16 @@ anyhow = "1"
cadence = "1" cadence = "1"
crates-index = { version = "2", default-features = false, features = ["git"] } crates-index = { version = "2", default-features = false, features = ["git"] }
derive_more = "0.99" derive_more = "0.99"
dotenvy = "0.15"
font-awesome-as-a-crate = "0.3" font-awesome-as-a-crate = "0.3"
futures-util = { version = "0.3", default-features = false, features = ["std"] } futures-util = { version = "0.3", default-features = false, features = ["std"] }
hyper = { version = "0.14.10", features = ["full"] } hyper = { version = "0.14.10", features = ["full"] }
error_reporter = "1"
indexmap = { version = "2", features = ["serde"] } indexmap = { version = "2", features = ["serde"] }
lru_time_cache = "0.11" lru_time_cache = "0.11"
maud = "0.26" maud = "0.26"
once_cell = "1" once_cell = "1"
parking_lot = "0.12"
pulldown-cmark = "0.11" pulldown-cmark = "0.11"
relative-path = { version = "1", features = ["serde"] } relative-path = { version = "1", features = ["serde"] }
reqwest = { version = "0.12", features = ["json"] } reqwest = { version = "0.12", features = ["json"] }
@ -33,11 +36,10 @@ rustsec = "0.29"
semver = { version = "1.0", features = ["serde"] } semver = { version = "1.0", features = ["serde"] }
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
serde_urlencoded = "0.7" serde_urlencoded = "0.7"
slog = "2"
slog-async = "2"
slog-term = "2"
tokio = { version = "1.24.2", features = ["rt-multi-thread", "macros", "sync", "time"] } tokio = { version = "1.24.2", features = ["rt-multi-thread", "macros", "sync", "time"] }
toml = "0.8" toml = "0.8"
tracing = "0.1.30"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
[target.'cfg(any())'.dependencies] [target.'cfg(any())'.dependencies]
gix = { version = "0.63", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls"] } gix = { version = "0.63", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls"] }

4
build.rs
View file

@ -1,6 +1,4 @@
use std::env; use std::{env, fs, path::Path};
use std::fs;
use std::path::Path;
use sha1::{Digest, Sha1}; use sha1::{Digest, Sha1};

4
libs/badge/badge.rs
View file

@ -235,8 +235,8 @@ mod tests {
#[test] #[test]
#[ignore] #[ignore]
fn test_to_svg() { fn test_to_svg() {
use std::fs::File; use std::{fs::File, io::Write as _};
use std::io::Write;
let mut file = File::create("test.svg").unwrap(); let mut file = File::create("test.svg").unwrap();
let options = BadgeOptions { let options = BadgeOptions {
subject: "latest".to_owned(), subject: "latest".to_owned(),

7
src/engine/fut/crawl.rs
View file

@ -2,11 +2,12 @@ use anyhow::Error;
use futures_util::{future::BoxFuture, stream::FuturesOrdered, FutureExt as _, StreamExt as _}; use futures_util::{future::BoxFuture, stream::FuturesOrdered, FutureExt as _, StreamExt as _};
use relative_path::RelativePathBuf; use relative_path::RelativePathBuf;
use crate::models::repo::RepoPath; use crate::{
engine::{
use crate::engine::{
machines::crawler::{ManifestCrawler, ManifestCrawlerOutput}, machines::crawler::{ManifestCrawler, ManifestCrawlerOutput},
Engine, Engine,
},
models::repo::RepoPath,
}; };
pub async fn crawl_manifest( pub async fn crawl_manifest(

3
src/engine/fut/mod.rs
View file

@ -1,5 +1,4 @@
mod analyze; mod analyze;
mod crawl; mod crawl;
pub use self::analyze::analyze_dependencies; pub use self::{analyze::analyze_dependencies, crawl::crawl_manifest};
pub use self::crawl::crawl_manifest;

3
src/engine/machines/analyzer.rs
View file

@ -101,9 +101,8 @@ impl DependencyAnalyzer {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::models::crates::{CrateDep, CrateDeps, CrateRelease};
use super::*; use super::*;
use crate::models::crates::{CrateDep, CrateDeps, CrateRelease};
#[test] #[test]
fn tracks_latest_without_matching() { fn tracks_latest_without_matching() {

9
src/engine/machines/crawler.rs
View file

@ -4,8 +4,10 @@ use anyhow::Error;
use indexmap::IndexMap; use indexmap::IndexMap;
use relative_path::RelativePathBuf; use relative_path::RelativePathBuf;
use crate::models::crates::{CrateDep, CrateDeps, CrateManifest, CrateName}; use crate::{
use crate::parsers::manifest::parse_manifest_toml; models::crates::{CrateDep, CrateDeps, CrateManifest, CrateName},
parsers::manifest::parse_manifest_toml,
};
pub struct ManifestCrawlerOutput { pub struct ManifestCrawlerOutput {
pub crates: IndexMap<CrateName, CrateDeps>, pub crates: IndexMap<CrateName, CrateDeps>,
@ -118,9 +120,8 @@ mod tests {
use relative_path::RelativePath; use relative_path::RelativePath;
use semver::VersionReq; use semver::VersionReq;
use crate::models::crates::CrateDep;
use super::*; use super::*;
use crate::models::crates::CrateDep;
#[test] #[test]
fn simple_package_manifest() { fn simple_package_manifest() {

36
src/engine/mod.rs
View file

@ -7,7 +7,6 @@ use std::{
use anyhow::{anyhow, Error}; use anyhow::{anyhow, Error};
use cadence::{MetricSink, NopMetricSink, StatsdClient}; use cadence::{MetricSink, NopMetricSink, StatsdClient};
use futures_util::{ use futures_util::{
future::try_join_all, future::try_join_all,
stream::{self, BoxStream}, stream::{self, BoxStream},
@ -18,16 +17,21 @@ use once_cell::sync::Lazy;
use relative_path::{RelativePath, RelativePathBuf}; use relative_path::{RelativePath, RelativePathBuf};
use rustsec::database::Database; use rustsec::database::Database;
use semver::VersionReq; use semver::VersionReq;
use slog::Logger;
use crate::interactors::crates::{GetPopularCrates, QueryCrate}; use crate::{
use crate::interactors::github::GetPopularRepos; interactors::{
use crate::interactors::rustsec::FetchAdvisoryDatabase; crates::{GetPopularCrates, QueryCrate},
use crate::interactors::RetrieveFileAtPath; github::GetPopularRepos,
use crate::models::crates::{AnalyzedDependencies, CrateName, CratePath, CrateRelease}; rustsec::FetchAdvisoryDatabase,
use crate::models::repo::{RepoPath, Repository}; RetrieveFileAtPath,
use crate::utils::cache::Cache; },
use crate::ManagedIndex; models::{
crates::{AnalyzedDependencies, CrateName, CratePath, CrateRelease},
repo::{RepoPath, Repository},
},
utils::cache::Cache,
ManagedIndex,
};
mod fut; mod fut;
mod machines; mod machines;
@ -45,33 +49,25 @@ pub struct Engine {
} }
impl Engine { impl Engine {
pub fn new(client: reqwest::Client, index: ManagedIndex, logger: Logger) -> Engine { pub fn new(client: reqwest::Client, index: ManagedIndex) -> Engine {
let metrics = Arc::new(StatsdClient::from_sink("engine", NopMetricSink)); let metrics = Arc::new(StatsdClient::from_sink("engine", NopMetricSink));
let query_crate = Cache::new( let query_crate = Cache::new(QueryCrate::new(index), Duration::from_secs(10), 500);
QueryCrate::new(index),
Duration::from_secs(10),
500,
logger.clone(),
);
let get_popular_crates = Cache::new( let get_popular_crates = Cache::new(
GetPopularCrates::new(client.clone()), GetPopularCrates::new(client.clone()),
Duration::from_secs(15 * 60), Duration::from_secs(15 * 60),
1, 1,
logger.clone(),
); );
let get_popular_repos = Cache::new( let get_popular_repos = Cache::new(
GetPopularRepos::new(client.clone()), GetPopularRepos::new(client.clone()),
Duration::from_secs(5 * 60), Duration::from_secs(5 * 60),
1, 1,
logger.clone(),
); );
let retrieve_file_at_path = RetrieveFileAtPath::new(client.clone()); let retrieve_file_at_path = RetrieveFileAtPath::new(client.clone());
let fetch_advisory_db = Cache::new( let fetch_advisory_db = Cache::new(
FetchAdvisoryDatabase::new(client), FetchAdvisoryDatabase::new(client),
Duration::from_secs(30 * 60), Duration::from_secs(30 * 60),
1, 1,
logger,
); );
Engine { Engine {

6
src/interactors/crates.rs
View file

@ -1,4 +1,7 @@
use std::{fmt, str, task::Context, task::Poll}; use std::{
fmt, str,
task::{Context, Poll},
};
use anyhow::{anyhow, Error}; use anyhow::{anyhow, Error};
use crates_index::{Crate, DependencyKind}; use crates_index::{Crate, DependencyKind};
@ -6,7 +9,6 @@ use futures_util::FutureExt as _;
use hyper::service::Service; use hyper::service::Service;
use semver::{Version, VersionReq}; use semver::{Version, VersionReq};
use serde::Deserialize; use serde::Deserialize;
use tokio::task::spawn_blocking; use tokio::task::spawn_blocking;
use crate::{ use crate::{

1
src/interactors/github.rs
View file

@ -4,7 +4,6 @@ use std::{
}; };
use anyhow::Error; use anyhow::Error;
use futures_util::FutureExt as _; use futures_util::FutureExt as _;
use hyper::service::Service; use hyper::service::Service;
use serde::Deserialize; use serde::Deserialize;

6
src/interactors/rustsec.rs
View file

@ -1,4 +1,8 @@
use std::{fmt, sync::Arc, task::Context, task::Poll}; use std::{
fmt,
sync::Arc,
task::{Context, Poll},
};
use anyhow::Error; use anyhow::Error;
use futures_util::FutureExt as _; use futures_util::FutureExt as _;

55
src/main.rs
View file

@ -15,9 +15,8 @@ use hyper::{
service::{make_service_fn, service_fn}, service::{make_service_fn, service_fn},
Server, Server,
}; };
use reqwest::redirect::Policy as RedirectPolicy; use reqwest::redirect::Policy as RedirectPolicy;
use slog::{error, info, o, Drain, Logger}; use tracing::Instrument as _;
mod engine; mod engine;
mod interactors; mod interactors;
@ -26,9 +25,7 @@ mod parsers;
mod server; mod server;
mod utils; mod utils;
use self::engine::Engine; use self::{engine::Engine, server::App, utils::index::ManagedIndex};
use self::server::App;
use self::utils::index::ManagedIndex;
/// Future crate's BoxFuture without the explicit lifetime parameter. /// Future crate's BoxFuture without the explicit lifetime parameter.
pub type BoxFuture<T> = Pin<Box<dyn Future<Output = T> + Send>>; pub type BoxFuture<T> = Pin<Box<dyn Future<Output = T> + Send>>;
@ -43,18 +40,29 @@ fn init_metrics() -> QueuingMetricSink {
QueuingMetricSink::from(sink) QueuingMetricSink::from(sink)
} }
fn init_root_logger() -> Logger { fn init_tracing_subscriber() {
let decorator = slog_term::TermDecorator::new().build(); use tracing::level_filters::LevelFilter;
let drain = slog_term::FullFormat::new(decorator).build().fuse(); use tracing_subscriber::{fmt, prelude::*, EnvFilter};
let drain = slog_async::Async::new(drain).build().fuse();
Logger::root(drain, o!()) let stdout_logger = match env::var("RUST_LOG_TIME").as_deref() {
Ok("false") => fmt::layer().without_time().boxed(),
_ => fmt::layer().boxed(),
};
tracing_subscriber::registry()
.with(
EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy(),
)
.with(stdout_logger)
.init();
} }
#[tokio::main] #[tokio::main]
async fn main() { async fn main() {
let logger = init_root_logger(); dotenvy::dotenv().ok();
init_tracing_subscriber();
let metrics = init_metrics(); let metrics = init_metrics();
let client = reqwest::Client::builder() let client = reqwest::Client::builder()
@ -71,7 +79,7 @@ async fn main() {
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port); let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port);
let index = ManagedIndex::new(logger.clone()); let index = ManagedIndex::new();
{ {
let index = index.clone(); let index = index.clone();
@ -81,27 +89,32 @@ async fn main() {
}); });
} }
let mut engine = Engine::new(client.clone(), index, logger.new(o!())); let mut engine = Engine::new(client.clone(), index);
engine.set_metrics(metrics); engine.set_metrics(metrics);
let svc_logger = logger.new(o!());
let make_svc = make_service_fn(move |_socket: &AddrStream| { let make_svc = make_service_fn(move |_socket: &AddrStream| {
let engine = engine.clone(); let engine = engine.clone();
let logger = svc_logger.clone();
async move { async move {
let server = App::new(logger.clone(), engine.clone()); let server = App::new(engine.clone());
Ok::<_, hyper::Error>(service_fn(move |req| { Ok::<_, hyper::Error>(service_fn(move |req| {
let server = server.clone(); let server = server.clone();
async move { server.handle(req).await } async move {
let path = req.uri().path().to_owned();
server
.handle(req)
.instrument(tracing::info_span!("@", %path))
.await
}
})) }))
} }
}); });
let server = Server::bind(&addr).serve(make_svc); let server = Server::bind(&addr).serve(make_svc);
info!(logger, "Server running on port {}", port); tracing::info!("Server running on port {port}");
if let Err(e) = server.await { if let Err(err) = server.await {
error!(logger, "server error: {}", e); tracing::error!("server error: {err}");
} }
} }

3
src/parsers/manifest.rs
View file

@ -135,9 +135,8 @@ pub fn parse_manifest_toml(input: &str) -> Result<CrateManifest, Error> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::models::crates::CrateManifest;
use super::*; use super::*;
use crate::models::crates::CrateManifest;
#[test] #[test]
fn parse_workspace_without_members_declaration() { fn parse_workspace_without_members_declaration() {

61
src/server/mod.rs
View file

@ -10,7 +10,6 @@ use once_cell::sync::Lazy;
use route_recognizer::{Params, Router}; use route_recognizer::{Params, Router};
use semver::VersionReq; use semver::VersionReq;
use serde::Deserialize; use serde::Deserialize;
use slog::{error, info, o, Logger};
mod assets; mod assets;
mod views; mod views;
@ -18,10 +17,14 @@ mod views;
use self::assets::{ use self::assets::{
STATIC_LINKS_JS_ETAG, STATIC_LINKS_JS_PATH, STATIC_STYLE_CSS_ETAG, STATIC_STYLE_CSS_PATH, STATIC_LINKS_JS_ETAG, STATIC_LINKS_JS_PATH, STATIC_STYLE_CSS_ETAG, STATIC_STYLE_CSS_PATH,
}; };
use crate::engine::{AnalyzeDependenciesOutcome, Engine}; use crate::{
use crate::models::crates::{CrateName, CratePath}; engine::{AnalyzeDependenciesOutcome, Engine},
use crate::models::repo::RepoPath; models::{
use crate::models::SubjectPath; crates::{CrateName, CratePath},
repo::RepoPath,
SubjectPath,
},
};
#[derive(Debug, Clone, Copy, PartialEq)] #[derive(Debug, Clone, Copy, PartialEq)]
enum StatusFormat { enum StatusFormat {
@ -47,13 +50,12 @@ enum Route {
#[derive(Clone)] #[derive(Clone)]
pub struct App { pub struct App {
logger: Logger,
engine: Engine, engine: Engine,
router: Arc<Router<Route>>, router: Arc<Router<Route>>,
} }
impl App { impl App {
pub fn new(logger: Logger, engine: Engine) -> App { pub fn new(engine: Engine) -> App {
let mut router = Router::new(); let mut router = Router::new();
router.add("/", Route::Index); router.add("/", Route::Index);
@ -83,15 +85,12 @@ impl App {
); );
App { App {
logger,
engine, engine,
router: Arc::new(router), router: Arc::new(router),
} }
} }
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, HyperError> { pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, HyperError> {
let logger = self.logger.new(o!("path" => req.uri().path().to_owned()));
let logger2 = logger.clone();
let start = Instant::now(); let start = Instant::now();
// allows `/path/` to also match `/path` // allows `/path/` to also match `/path`
@ -99,28 +98,25 @@ impl App {
let res = if let Ok(route_match) = self.router.recognize(normalized_path) { let res = if let Ok(route_match) = self.router.recognize(normalized_path) {
match (req.method(), route_match.handler()) { match (req.method(), route_match.handler()) {
(&Method::GET, Route::Index) => { (&Method::GET, Route::Index) => self.index(req, route_match.params().clone()).await,
self.index(req, route_match.params().clone(), logger).await
}
(&Method::GET, Route::RepoStatus(format)) => { (&Method::GET, Route::RepoStatus(format)) => {
self.repo_status(req, route_match.params().clone(), logger, *format) self.repo_status(req, route_match.params().clone(), *format)
.await .await
} }
(&Method::GET, Route::CrateStatus(format)) => { (&Method::GET, Route::CrateStatus(format)) => {
self.crate_status(req, route_match.params().clone(), logger, *format) self.crate_status(req, route_match.params().clone(), *format)
.await .await
} }
(&Method::GET, Route::LatestCrateBadge) => { (&Method::GET, Route::LatestCrateBadge) => {
self.crate_status(req, route_match.params().clone(), logger, StatusFormat::Svg) self.crate_status(req, route_match.params().clone(), StatusFormat::Svg)
.await .await
} }
(&Method::GET, Route::CrateRedirect) => { (&Method::GET, Route::CrateRedirect) => {
self.crate_redirect(req, route_match.params().clone(), logger) self.crate_redirect(req, route_match.params().clone()).await
.await
} }
(&Method::GET, Route::Static(file)) => Ok(App::static_file(*file)), (&Method::GET, Route::Static(file)) => Ok(App::static_file(*file)),
@ -135,12 +131,11 @@ impl App {
let diff = end - start; let diff = end - start;
match &res { match &res {
Ok(res) => info!( Ok(res) => tracing::info!(
logger2, ""; status = %res.status(),
"status" => res.status().to_string(), time = %format_args!("{}ms", diff.as_millis()),
"time" => format!("{}ms", diff.as_millis())
), ),
Err(err) => error!(logger2, ""; "error" => err.to_string()), Err(err) => tracing::error!(%err),
}; };
res res
@ -152,7 +147,6 @@ impl App {
&self, &self,
_req: Request<Body>, _req: Request<Body>,
_params: Params, _params: Params,
logger: Logger,
) -> Result<Response<Body>, HyperError> { ) -> Result<Response<Body>, HyperError> {
let engine = self.engine.clone(); let engine = self.engine.clone();
@ -161,7 +155,7 @@ impl App {
match popular { match popular {
Err(err) => { Err(err) => {
error!(logger, "error: {}", err); tracing::error!(%err);
let mut response = let mut response =
views::html::error::render("Could not retrieve popular items", ""); views::html::error::render("Could not retrieve popular items", "");
*response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR;
@ -177,7 +171,6 @@ impl App {
&self, &self,
req: Request<Body>, req: Request<Body>,
params: Params, params: Params,
logger: Logger,
format: StatusFormat, format: StatusFormat,
) -> Result<Response<Body>, HyperError> { ) -> Result<Response<Body>, HyperError> {
let server = self.clone(); let server = self.clone();
@ -192,7 +185,7 @@ impl App {
match repo_path_result { match repo_path_result {
Err(err) => { Err(err) => {
error!(logger, "error: {}", err); tracing::error!(%err);
let mut response = views::html::error::render( let mut response = views::html::error::render(
"Could not parse repository path", "Could not parse repository path",
"Please make sure to provide a valid repository path.", "Please make sure to provide a valid repository path.",
@ -209,7 +202,7 @@ impl App {
match analyze_result { match analyze_result {
Err(err) => { Err(err) => {
error!(logger, "error: {}", err); tracing::error!(%err);
let response = App::status_format_analysis( let response = App::status_format_analysis(
None, None,
format, format,
@ -236,7 +229,6 @@ impl App {
&self, &self,
_req: Request<Body>, _req: Request<Body>,
params: Params, params: Params,
logger: Logger,
) -> Result<Response<Body>, HyperError> { ) -> Result<Response<Body>, HyperError> {
let engine = self.engine.clone(); let engine = self.engine.clone();
@ -245,7 +237,7 @@ impl App {
match crate_name_result { match crate_name_result {
Err(err) => { Err(err) => {
error!(logger, "error: {}", err); tracing::error!(%err);
let mut response = views::html::error::render( let mut response = views::html::error::render(
"Could not parse crate name", "Could not parse crate name",
"Please make sure to provide a valid crate name.", "Please make sure to provide a valid crate name.",
@ -261,7 +253,7 @@ impl App {
match release_result { match release_result {
Err(err) => { Err(err) => {
error!(logger, "error: {}", err); tracing::error!(%err);
let mut response = views::html::error::render( let mut response = views::html::error::render(
"Could not fetch crate information", "Could not fetch crate information",
"Please make sure to provide a valid crate name.", "Please make sure to provide a valid crate name.",
@ -302,7 +294,6 @@ impl App {
&self, &self,
req: Request<Body>, req: Request<Body>,
params: Params, params: Params,
logger: Logger,
format: StatusFormat, format: StatusFormat,
) -> Result<Response<Body>, HyperError> { ) -> Result<Response<Body>, HyperError> {
let server = self.clone(); let server = self.clone();
@ -332,7 +323,7 @@ impl App {
Ok(Some(latest_rel)) => latest_rel.version.to_string(), Ok(Some(latest_rel)) => latest_rel.version.to_string(),
Ok(None) => return Ok(not_found()), Ok(None) => return Ok(not_found()),
Err(err) => { Err(err) => {
error!(logger, "error: {}", err); tracing::error!(%err);
let mut response = views::html::error::render( let mut response = views::html::error::render(
"Could not fetch crate information", "Could not fetch crate information",
"Please make sure to provide a valid crate name.", "Please make sure to provide a valid crate name.",
@ -349,7 +340,7 @@ impl App {
match crate_path_result { match crate_path_result {
Err(err) => { Err(err) => {
error!(logger, "error: {}", err); tracing::error!(%err);
let mut response = views::html::error::render( let mut response = views::html::error::render(
"Could not parse crate path", "Could not parse crate path",
"Please make sure to provide a valid crate name and version.", "Please make sure to provide a valid crate name and version.",
@ -365,7 +356,7 @@ impl App {
match analyze_result { match analyze_result {
Err(err) => { Err(err) => {
error!(logger, "error: {}", err); tracing::error!(%err);
let response = App::status_format_analysis( let response = App::status_format_analysis(
None, None,
format, format,

6
src/server/views/badge.rs
View file

@ -1,9 +1,7 @@
use badge::{Badge, BadgeOptions}; use badge::{Badge, BadgeOptions};
use hyper::header::CONTENT_TYPE; use hyper::{header::CONTENT_TYPE, Body, Response};
use hyper::{Body, Response};
use crate::engine::AnalyzeDependenciesOutcome; use crate::{engine::AnalyzeDependenciesOutcome, server::ExtraConfig};
use crate::server::ExtraConfig;
pub fn badge( pub fn badge(
analysis_outcome: Option<&AnalyzeDependenciesOutcome>, analysis_outcome: Option<&AnalyzeDependenciesOutcome>,

8
src/server/views/html/index.rs
View file

@ -1,10 +1,10 @@
use hyper::{Body, Response}; use hyper::{Body, Response};
use maud::{html, Markup}; use maud::{html, Markup};
use crate::models::crates::CratePath; use crate::{
use crate::models::repo::Repository; models::{crates::CratePath, repo::Repository},
server::assets::STATIC_LINKS_JS_PATH,
use crate::server::assets::STATIC_LINKS_JS_PATH; };
fn link_forms() -> Markup { fn link_forms() -> Markup {
html! { html! {

6
src/server/views/html/mod.rs
View file

@ -1,15 +1,13 @@
use std::time::Duration; use std::time::Duration;
use hyper::header::CONTENT_TYPE; use hyper::{header::CONTENT_TYPE, Body, Response};
use hyper::{Body, Response};
use maud::{html, Markup, Render, DOCTYPE}; use maud::{html, Markup, Render, DOCTYPE};
pub mod error; pub mod error;
pub mod index; pub mod index;
pub mod status; pub mod status;
use crate::server::assets::STATIC_STYLE_CSS_PATH; use crate::server::{assets::STATIC_STYLE_CSS_PATH, SELF_BASE_URL};
use crate::server::SELF_BASE_URL;
fn render_html<B: Render>(title: &str, body: B) -> Response<Body> { fn render_html<B: Render>(title: &str, body: B) -> Response<Body> {
let rendered = html! { let rendered = html! {

15
src/server/views/html/status.rs
View file

@ -6,12 +6,15 @@ use pulldown_cmark::{html, Parser};
use rustsec::advisory::Advisory; use rustsec::advisory::Advisory;
use semver::Version; use semver::Version;
use crate::engine::AnalyzeDependenciesOutcome; use crate::{
use crate::models::crates::{AnalyzedDependencies, AnalyzedDependency, CrateName}; engine::AnalyzeDependenciesOutcome,
use crate::models::repo::RepoSite; models::{
use crate::models::SubjectPath; crates::{AnalyzedDependencies, AnalyzedDependency, CrateName},
use crate::server::views::badge; repo::RepoSite,
use crate::server::ExtraConfig; SubjectPath,
},
server::{views::badge, ExtraConfig},
};
fn get_crates_url(name: impl AsRef<str>) -> String { fn get_crates_url(name: impl AsRef<str>) -> String {
format!("https://crates.io/crates/{}", name.as_ref()) format!("https://crates.io/crates/{}", name.as_ref())

21
src/utils/cache.rs
View file

@ -3,7 +3,6 @@ use std::{fmt, sync::Arc, time::Duration};
use derive_more::{Display, Error, From}; use derive_more::{Display, Error, From};
use hyper::service::Service; use hyper::service::Service;
use lru_time_cache::LruCache; use lru_time_cache::LruCache;
use slog::{debug, Logger};
use tokio::sync::Mutex; use tokio::sync::Mutex;
#[derive(Debug, Clone, Display, From, Error)] #[derive(Debug, Clone, Display, From, Error)]
@ -18,7 +17,6 @@ where
{ {
inner: S, inner: S,
cache: Arc<Mutex<LruCache<Req, S::Response>>>, cache: Arc<Mutex<LruCache<Req, S::Response>>>,
logger: Logger,
} }
impl<S, Req> fmt::Debug for Cache<S, Req> impl<S, Req> fmt::Debug for Cache<S, Req>
@ -38,13 +36,12 @@ where
S::Response: Clone, S::Response: Clone,
Req: Clone + Eq + Ord + fmt::Debug, Req: Clone + Eq + Ord + fmt::Debug,
{ {
pub fn new(service: S, ttl: Duration, capacity: usize, logger: Logger) -> Cache<S, Req> { pub fn new(service: S, ttl: Duration, capacity: usize) -> Cache<S, Req> {
let cache = LruCache::with_expiry_duration_and_capacity(ttl, capacity); let cache = LruCache::with_expiry_duration_and_capacity(ttl, capacity);
Cache { Cache {
inner: service, inner: service,
cache: Arc::new(Mutex::new(cache)), cache: Arc::new(Mutex::new(cache)),
logger,
} }
} }
@ -53,19 +50,19 @@ where
let mut cache = self.cache.lock().await; let mut cache = self.cache.lock().await;
if let Some(cached_response) = cache.get(&req) { if let Some(cached_response) = cache.get(&req) {
debug!( tracing::debug!(
self.logger, "cache hit"; svc = ?self.inner,
"svc" => format!("{:?}", self.inner), req = ?req,
"req" => format!("{:?}", &req) cache = "hit",
); );
return Ok(cached_response.clone()); return Ok(cached_response.clone());
} }
} }
debug!( tracing::debug!(
self.logger, "cache miss"; svc = ?self.inner,
"svc" => format!("{:?}", self.inner), req = ?req,
"req" => format!("{:?}", &req) cache = "miss",
); );
let mut service = self.inner.clone(); let mut service = self.inner.clone();

46
src/utils/index.rs
View file

@ -1,32 +1,30 @@
use std::sync::Arc; use std::{sync::Arc, time::Duration};
use std::sync::Mutex;
use std::time::Duration; use anyhow::Result;
use crates_index::{Crate, GitIndex};
use parking_lot::Mutex;
use tokio::{
task::spawn_blocking,
time::{self, MissedTickBehavior},
};
use crate::models::crates::CrateName; use crate::models::crates::CrateName;
use anyhow::Result;
use crates_index::Crate;
use crates_index::GitIndex;
use slog::{error, Logger};
use tokio::task::spawn_blocking;
use tokio::time::{self, MissedTickBehavior};
#[derive(Clone)] #[derive(Clone)]
pub struct ManagedIndex { pub struct ManagedIndex {
index: Arc<Mutex<GitIndex>>, index: Arc<Mutex<GitIndex>>,
logger: Logger,
} }
impl ManagedIndex { impl ManagedIndex {
pub fn new(logger: Logger) -> Self { pub fn new() -> Self {
// the index path is configurable through the `CARGO_HOME` env variable // the index path is configurable through the `CARGO_HOME` env variable
let index = Arc::new(Mutex::new(GitIndex::new_cargo_default().unwrap())); let index = Arc::new(Mutex::new(GitIndex::new_cargo_default().unwrap()));
Self { index, logger }
Self { index }
} }
pub fn crate_(&self, crate_name: CrateName) -> Option<Crate> { pub fn crate_(&self, crate_name: CrateName) -> Option<Crate> {
let index = self.index.lock().unwrap(); self.index.lock().crate_(crate_name.as_ref())
index.crate_(crate_name.as_ref())
} }
pub async fn refresh_at_interval(&self, update_interval: Duration) { pub async fn refresh_at_interval(&self, update_interval: Duration) {
@ -34,25 +32,23 @@ impl ManagedIndex {
update_interval.set_missed_tick_behavior(MissedTickBehavior::Delay); update_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
loop { loop {
if let Err(e) = self.refresh().await { if let Err(err) = self.refresh().await {
error!( tracing::error!(
self.logger, "failed refreshing the crates.io-index, the operation will be retried: {}",
"failed refreshing the crates.io-index, the operation will be retried: {}", e error_reporter::Report::new(err),
); );
} }
update_interval.tick().await; update_interval.tick().await;
} }
} }
async fn refresh(&self) -> Result<()> { async fn refresh(&self) -> Result<(), crates_index::Error> {
let index = Arc::clone(&self.index); let index = Arc::clone(&self.index);
spawn_blocking(move || { spawn_blocking(move || index.lock().update())
let mut index = index.lock().unwrap(); .await
.expect("blocking index update task should never panic")?;
index.update()
})
.await??;
Ok(()) Ok(())
} }
} }