Compare commits

..

2 commits

Author SHA1 Message Date
Rob Ede
b03684894e
feat: migrate web server to axum 2024-05-26 22:30:06 +01:00
Rob Ede
ebec04d2de
refactor: migrate to tracing for logging 2024-05-26 21:42:38 +01:00
21 changed files with 78 additions and 96 deletions

View file

@ -1,3 +0,0 @@
imports_granularity = "Crate"
group_imports = "StdExternalCrate"
use_field_init_shorthand = true

8
Cargo.lock generated
View file

@ -497,12 +497,6 @@ dependencies = [
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
[[package]]
name = "error_reporter"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31ae425815400e5ed474178a7a22e275a9687086a12ca63ec793ff292d8fdae8"
[[package]] [[package]]
name = "faster-hex" name = "faster-hex"
version = "0.9.0" version = "0.9.0"
@ -2634,7 +2628,6 @@ dependencies = [
"crates-index", "crates-index",
"derive_more", "derive_more",
"dotenvy", "dotenvy",
"error_reporter",
"font-awesome-as-a-crate", "font-awesome-as-a-crate",
"futures-util", "futures-util",
"gix", "gix",
@ -2643,7 +2636,6 @@ dependencies = [
"lru_time_cache", "lru_time_cache",
"maud", "maud",
"once_cell", "once_cell",
"parking_lot",
"pulldown-cmark", "pulldown-cmark",
"relative-path", "relative-path",
"reqwest", "reqwest",

View file

@ -22,12 +22,10 @@ derive_more = "0.99"
dotenvy = "0.15" dotenvy = "0.15"
font-awesome-as-a-crate = "0.3" font-awesome-as-a-crate = "0.3"
futures-util = { version = "0.3", default-features = false, features = ["std"] } futures-util = { version = "0.3", default-features = false, features = ["std"] }
error_reporter = "1"
indexmap = { version = "2", features = ["serde"] } indexmap = { version = "2", features = ["serde"] }
lru_time_cache = "0.11" lru_time_cache = "0.11"
maud = "0.26" maud = "0.26"
once_cell = "1" once_cell = "1"
parking_lot = "0.12"
pulldown-cmark = "0.11" pulldown-cmark = "0.11"
relative-path = { version = "1", features = ["serde"] } relative-path = { version = "1", features = ["serde"] }
reqwest = { version = "0.12", features = ["json"] } reqwest = { version = "0.12", features = ["json"] }

View file

@ -1,4 +1,6 @@
use std::{env, fs, path::Path}; use std::env;
use std::fs;
use std::path::Path;
use sha1::{Digest, Sha1}; use sha1::{Digest, Sha1};

View file

@ -235,8 +235,8 @@ mod tests {
#[test] #[test]
#[ignore] #[ignore]
fn test_to_svg() { fn test_to_svg() {
use std::{fs::File, io::Write as _}; use std::fs::File;
use std::io::Write;
let mut file = File::create("test.svg").unwrap(); let mut file = File::create("test.svg").unwrap();
let options = BadgeOptions { let options = BadgeOptions {
subject: "latest".to_owned(), subject: "latest".to_owned(),

View file

@ -2,12 +2,11 @@ use anyhow::Error;
use futures_util::{future::BoxFuture, stream::FuturesOrdered, FutureExt as _, StreamExt as _}; use futures_util::{future::BoxFuture, stream::FuturesOrdered, FutureExt as _, StreamExt as _};
use relative_path::RelativePathBuf; use relative_path::RelativePathBuf;
use crate::{ use crate::models::repo::RepoPath;
engine::{
machines::crawler::{ManifestCrawler, ManifestCrawlerOutput}, use crate::engine::{
Engine, machines::crawler::{ManifestCrawler, ManifestCrawlerOutput},
}, Engine,
models::repo::RepoPath,
}; };
pub async fn crawl_manifest( pub async fn crawl_manifest(

View file

@ -1,4 +1,5 @@
mod analyze; mod analyze;
mod crawl; mod crawl;
pub use self::{analyze::analyze_dependencies, crawl::crawl_manifest}; pub use self::analyze::analyze_dependencies;
pub use self::crawl::crawl_manifest;

View file

@ -101,9 +101,10 @@ impl DependencyAnalyzer {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*;
use crate::models::crates::{CrateDep, CrateDeps, CrateRelease}; use crate::models::crates::{CrateDep, CrateDeps, CrateRelease};
use super::*;
#[test] #[test]
fn tracks_latest_without_matching() { fn tracks_latest_without_matching() {
let mut deps = CrateDeps::default(); let mut deps = CrateDeps::default();

View file

@ -4,10 +4,8 @@ use anyhow::Error;
use indexmap::IndexMap; use indexmap::IndexMap;
use relative_path::RelativePathBuf; use relative_path::RelativePathBuf;
use crate::{ use crate::models::crates::{CrateDep, CrateDeps, CrateManifest, CrateName};
models::crates::{CrateDep, CrateDeps, CrateManifest, CrateName}, use crate::parsers::manifest::parse_manifest_toml;
parsers::manifest::parse_manifest_toml,
};
pub struct ManifestCrawlerOutput { pub struct ManifestCrawlerOutput {
pub crates: IndexMap<CrateName, CrateDeps>, pub crates: IndexMap<CrateName, CrateDeps>,
@ -120,9 +118,10 @@ mod tests {
use relative_path::RelativePath; use relative_path::RelativePath;
use semver::VersionReq; use semver::VersionReq;
use super::*;
use crate::models::crates::CrateDep; use crate::models::crates::CrateDep;
use super::*;
#[test] #[test]
fn simple_package_manifest() { fn simple_package_manifest() {
let manifest = r#" let manifest = r#"

View file

@ -7,6 +7,7 @@ use std::{
use anyhow::{anyhow, Error}; use anyhow::{anyhow, Error};
use cadence::{MetricSink, NopMetricSink, StatsdClient}; use cadence::{MetricSink, NopMetricSink, StatsdClient};
use futures_util::{ use futures_util::{
future::try_join_all, future::try_join_all,
stream::{self, BoxStream}, stream::{self, BoxStream},
@ -18,20 +19,14 @@ use rustsec::database::Database;
use semver::VersionReq; use semver::VersionReq;
use tower::Service; use tower::Service;
use crate::{ use crate::interactors::crates::{GetPopularCrates, QueryCrate};
interactors::{ use crate::interactors::github::GetPopularRepos;
crates::{GetPopularCrates, QueryCrate}, use crate::interactors::rustsec::FetchAdvisoryDatabase;
github::GetPopularRepos, use crate::interactors::RetrieveFileAtPath;
rustsec::FetchAdvisoryDatabase, use crate::models::crates::{AnalyzedDependencies, CrateName, CratePath, CrateRelease};
RetrieveFileAtPath, use crate::models::repo::{RepoPath, Repository};
}, use crate::utils::cache::Cache;
models::{ use crate::ManagedIndex;
crates::{AnalyzedDependencies, CrateName, CratePath, CrateRelease},
repo::{RepoPath, Repository},
},
utils::cache::Cache,
ManagedIndex,
};
mod fut; mod fut;
mod machines; mod machines;

View file

@ -1,16 +1,14 @@
use std::{ use std::{fmt, str, task::Context, task::Poll};
fmt, str,
task::{Context, Poll},
};
use anyhow::{anyhow, Error}; use anyhow::{anyhow, Error};
use crates_index::{Crate, DependencyKind}; use crates_index::{Crate, DependencyKind};
use futures_util::FutureExt as _; use futures_util::FutureExt as _;
use semver::{Version, VersionReq}; use semver::{Version, VersionReq};
use serde::Deserialize; use serde::Deserialize;
use tokio::task::spawn_blocking;
use tower::Service; use tower::Service;
use tokio::task::spawn_blocking;
use crate::{ use crate::{
models::crates::{CrateDep, CrateDeps, CrateName, CratePath, CrateRelease}, models::crates::{CrateDep, CrateDeps, CrateName, CratePath, CrateRelease},
BoxFuture, ManagedIndex, BoxFuture, ManagedIndex,

View file

@ -4,6 +4,7 @@ use std::{
}; };
use anyhow::Error; use anyhow::Error;
use futures_util::FutureExt as _; use futures_util::FutureExt as _;
use serde::Deserialize; use serde::Deserialize;
use tower::Service; use tower::Service;

View file

@ -1,8 +1,4 @@
use std::{ use std::{fmt, sync::Arc, task::Context, task::Poll};
fmt,
sync::Arc,
task::{Context, Poll},
};
use anyhow::Error; use anyhow::Error;
use futures_util::FutureExt as _; use futures_util::FutureExt as _;

View file

@ -21,7 +21,9 @@ mod parsers;
mod server; mod server;
mod utils; mod utils;
use self::{engine::Engine, server::App, utils::index::ManagedIndex}; use self::engine::Engine;
use self::server::App;
use self::utils::index::ManagedIndex;
/// Future crate's BoxFuture without the explicit lifetime parameter. /// Future crate's BoxFuture without the explicit lifetime parameter.
pub type BoxFuture<T> = Pin<Box<dyn Future<Output = T> + Send>>; pub type BoxFuture<T> = Pin<Box<dyn Future<Output = T> + Send>>;

View file

@ -135,9 +135,10 @@ pub fn parse_manifest_toml(input: &str) -> Result<CrateManifest, Error> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*;
use crate::models::crates::CrateManifest; use crate::models::crates::CrateManifest;
use super::*;
#[test] #[test]
fn parse_workspace_without_members_declaration() { fn parse_workspace_without_members_declaration() {
let toml = r#"[package] let toml = r#"[package]

View file

@ -22,14 +22,10 @@ mod views;
use self::assets::{ use self::assets::{
STATIC_LINKS_JS_ETAG, STATIC_LINKS_JS_PATH, STATIC_STYLE_CSS_ETAG, STATIC_STYLE_CSS_PATH, STATIC_LINKS_JS_ETAG, STATIC_LINKS_JS_PATH, STATIC_STYLE_CSS_ETAG, STATIC_STYLE_CSS_PATH,
}; };
use crate::{ use crate::engine::{AnalyzeDependenciesOutcome, Engine};
engine::{AnalyzeDependenciesOutcome, Engine}, use crate::models::crates::{CrateName, CratePath};
models::{ use crate::models::repo::RepoPath;
crates::{CrateName, CratePath}, use crate::models::SubjectPath;
repo::RepoPath,
SubjectPath,
},
};
#[derive(Debug, Clone, Copy, PartialEq)] #[derive(Debug, Clone, Copy, PartialEq)]
enum StatusFormat { enum StatusFormat {

View file

@ -1,7 +1,10 @@
use axum::{body::Body, http::header::CONTENT_TYPE, response::Response}; use axum::body::Body;
use axum::http::header::CONTENT_TYPE;
use axum::response::Response;
use badge::{Badge, BadgeOptions}; use badge::{Badge, BadgeOptions};
use crate::{engine::AnalyzeDependenciesOutcome, server::ExtraConfig}; use crate::engine::AnalyzeDependenciesOutcome;
use crate::server::ExtraConfig;
pub fn badge( pub fn badge(
analysis_outcome: Option<&AnalyzeDependenciesOutcome>, analysis_outcome: Option<&AnalyzeDependenciesOutcome>,

View file

@ -1,10 +1,10 @@
use axum::{body::Body, response::Response}; use axum::{body::Body, response::Response};
use maud::{html, Markup}; use maud::{html, Markup};
use crate::{ use crate::models::crates::CratePath;
models::{crates::CratePath, repo::Repository}, use crate::models::repo::Repository;
server::assets::STATIC_LINKS_JS_PATH,
}; use crate::server::assets::STATIC_LINKS_JS_PATH;
fn link_forms() -> Markup { fn link_forms() -> Markup {
html! { html! {

View file

@ -1,13 +1,15 @@
use std::time::Duration; use std::time::Duration;
use axum::{body::Body, http::header::CONTENT_TYPE, response::Response}; use axum::http::header::CONTENT_TYPE;
use axum::{body::Body, response::Response};
use maud::{html, Markup, Render, DOCTYPE}; use maud::{html, Markup, Render, DOCTYPE};
pub mod error; pub mod error;
pub mod index; pub mod index;
pub mod status; pub mod status;
use crate::server::{assets::STATIC_STYLE_CSS_PATH, SELF_BASE_URL}; use crate::server::assets::STATIC_STYLE_CSS_PATH;
use crate::server::SELF_BASE_URL;
fn render_html<B: Render>(title: &str, body: B) -> Response<Body> { fn render_html<B: Render>(title: &str, body: B) -> Response<Body> {
let rendered = html! { let rendered = html! {

View file

@ -6,15 +6,12 @@ use pulldown_cmark::{html, Parser};
use rustsec::advisory::Advisory; use rustsec::advisory::Advisory;
use semver::Version; use semver::Version;
use crate::{ use crate::engine::AnalyzeDependenciesOutcome;
engine::AnalyzeDependenciesOutcome, use crate::models::crates::{AnalyzedDependencies, AnalyzedDependency, CrateName};
models::{ use crate::models::repo::RepoSite;
crates::{AnalyzedDependencies, AnalyzedDependency, CrateName}, use crate::models::SubjectPath;
repo::RepoSite, use crate::server::views::badge;
SubjectPath, use crate::server::ExtraConfig;
},
server::{views::badge, ExtraConfig},
};
fn get_crates_url(name: impl AsRef<str>) -> String { fn get_crates_url(name: impl AsRef<str>) -> String {
format!("https://crates.io/crates/{}", name.as_ref()) format!("https://crates.io/crates/{}", name.as_ref())

View file

@ -1,14 +1,13 @@
use std::{sync::Arc, time::Duration}; use std::sync::Arc;
use std::sync::Mutex;
use anyhow::Result; use std::time::Duration;
use crates_index::{Crate, GitIndex};
use parking_lot::Mutex;
use tokio::{
task::spawn_blocking,
time::{self, MissedTickBehavior},
};
use crate::models::crates::CrateName; use crate::models::crates::CrateName;
use anyhow::Result;
use crates_index::Crate;
use crates_index::GitIndex;
use tokio::task::spawn_blocking;
use tokio::time::{self, MissedTickBehavior};
#[derive(Clone)] #[derive(Clone)]
pub struct ManagedIndex { pub struct ManagedIndex {
@ -24,7 +23,9 @@ impl ManagedIndex {
} }
pub fn crate_(&self, crate_name: CrateName) -> Option<Crate> { pub fn crate_(&self, crate_name: CrateName) -> Option<Crate> {
self.index.lock().crate_(crate_name.as_ref()) let index = self.index.lock().unwrap();
index.crate_(crate_name.as_ref())
} }
pub async fn refresh_at_interval(&self, update_interval: Duration) { pub async fn refresh_at_interval(&self, update_interval: Duration) {
@ -34,21 +35,22 @@ impl ManagedIndex {
loop { loop {
if let Err(err) = self.refresh().await { if let Err(err) = self.refresh().await {
tracing::error!( tracing::error!(
"failed refreshing the crates.io-index, the operation will be retried: {}", "failed refreshing the crates.io-index, the operation will be retried: {err}"
error_reporter::Report::new(err),
); );
} }
update_interval.tick().await; update_interval.tick().await;
} }
} }
async fn refresh(&self) -> Result<(), crates_index::Error> { async fn refresh(&self) -> Result<()> {
let index = Arc::clone(&self.index); let index = Arc::clone(&self.index);
spawn_blocking(move || index.lock().update()) spawn_blocking(move || {
.await let mut index = index.lock().unwrap();
.expect("blocking index update task should never panic")?;
index.update()
})
.await??;
Ok(()) Ok(())
} }
} }