From c4337ae69842ed97f78585020841f8f72d273517 Mon Sep 17 00:00:00 2001 From: Sam Rijs Date: Sat, 24 Feb 2018 17:37:42 +1100 Subject: [PATCH] instrument analyze timings --- Cargo.lock | 16 ++++++++++++++++ Cargo.toml | 1 + src/engine/mod.rs | 27 +++++++++++++++++++++------ src/main.rs | 17 +++++++++++++++-- 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8205630..2d8b30c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,6 +68,14 @@ dependencies = [ "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "cadence" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "cc" version = "1.0.4" @@ -104,6 +112,11 @@ dependencies = [ "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "crossbeam" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "dtoa" version = "0.4.2" @@ -682,6 +695,7 @@ name = "shiny-robots" version = "0.1.0" dependencies = [ "badge 0.2.0", + "cadence 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.11.18 (registry+https://github.com/rust-lang/crates.io-index)", @@ -947,11 +961,13 @@ dependencies = [ "checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf" "checksum byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "652805b7e73fada9d85e9a6682a4abd490cb52d96aeecc12e33a0de34dfd0d23" "checksum bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1b7db437d718977f6dc9b2e3fd6fc343c02ac6b899b73fdd2179163447bd9ce9" +"checksum cadence 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f1725ad9d5c8251726f1d7207d49e635e92448010fca03bddc7f07f0a4488a33" "checksum cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "deaf9ec656256bb25b404c51ef50097207b9cbb29c933d31f92cae5a8a0ffee0" "checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" "checksum chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c20ebe0b2b08b0aeddba49c609fe7957ba2e33449882cb186a180bc60682fa9" "checksum core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25bfd746d203017f7d5cbd31ee5d8e17f94b6521c7af77ece6c9e4b2d4b16c67" "checksum core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "065a5d7ffdcbc8fa145d6f0746f3555025b9097a9e9cda59f7467abae670c78d" +"checksum crossbeam 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "bd66663db5a988098a89599d4857919b3acf7f61402e61365acfd3919857b9be" "checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab" "checksum failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "934799b6c1de475a012a02dab0ace1ace43789ee4b99bcfbf1a2e3e8ced5de82" "checksum failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c7cdda555bb90c9bb67a3b670a0f42de8e73f5981524123ad8578aafec8ddb8b" diff --git a/Cargo.toml b/Cargo.toml index bfd8c46..46ebd41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" authors = ["Sam Rijs "] [dependencies] +cadence = "0.13.1" failure = "0.1.1" futures = "0.1.18" hyper = "0.11.15" diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 52cc80b..ba6f76d 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -2,6 +2,8 @@ use std::collections::HashSet; use std::sync::Arc; use std::time::{Duration, Instant}; +use cadence::prelude::*; +use cadence::{MetricSink, NopMetricSink, StatsdClient}; use failure::Error; use futures::{Future, future}; use futures::future::join_all; @@ -34,6 +36,7 @@ type HttpClient = Client>; pub struct Engine { client: HttpClient, logger: Logger, + metrics: StatsdClient, query_crate: Arc>>, get_popular_crates: Arc>>, @@ -43,12 +46,14 @@ pub struct Engine { impl Engine { pub fn new(client: Client>, logger: Logger) -> Engine { + let metrics = StatsdClient::from_sink("engine", NopMetricSink); + let query_crate = Cache::new(QueryCrate(client.clone()), Duration::from_secs(300), 500); let get_popular_crates = Cache::new(GetPopularCrates(client.clone()), Duration::from_secs(10), 1); let get_popular_repos = Cache::new(GetPopularRepos(client.clone()), Duration::from_secs(10), 1); Engine { - client: client.clone(), logger, + client: client.clone(), logger, metrics, query_crate: Arc::new(query_crate), get_popular_crates: Arc::new(get_popular_crates), @@ -56,6 +61,10 @@ impl Engine { retrieve_file_at_path: Arc::new(RetrieveFileAtPath(client)) } } + + pub fn set_metrics(&mut self, sink: M) { + self.metrics = StatsdClient::from_sink("engine", sink); + } } pub struct AnalyzeDependenciesOutcome { @@ -100,22 +109,28 @@ impl Engine { let start = Instant::now(); let entry_point = RelativePath::new("/").to_relative_path_buf(); - let manifest_future = CrawlManifestFuture::new(self, repo_path, entry_point); + let manifest_future = CrawlManifestFuture::new(self, repo_path.clone(), entry_point); let engine = self.clone(); manifest_future.and_then(move |manifest_output| { + let engine_for_analyze = engine.clone(); let futures = manifest_output.crates.into_iter().map(move |(crate_name, deps)| { - let analyzed_deps_future = AnalyzeDependenciesFuture::new(&engine, deps); + let analyzed_deps_future = AnalyzeDependenciesFuture::new(&engine_for_analyze, deps); analyzed_deps_future.map(move |analyzed_deps| (crate_name, analyzed_deps)) }); - join_all(futures).map(move |crates| { + join_all(futures).and_then(move |crates| { let duration = start.elapsed(); + engine.metrics.time_duration_with_tags("analyze_duration", duration) + .with_tag("repo_site", repo_path.site.as_ref()) + .with_tag("repo_qual", repo_path.qual.as_ref()) + .with_tag("repo_name", repo_path.name.as_ref()) + .send()?; - AnalyzeDependenciesOutcome { + Ok(AnalyzeDependenciesOutcome { crates, duration - } + }) }) }) } diff --git a/src/main.rs b/src/main.rs index 8ea8e0f..5584621 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ #![feature(proc_macro)] extern crate badge; +extern crate cadence; #[macro_use] extern crate failure; #[macro_use] extern crate futures; extern crate hyper; @@ -33,9 +34,10 @@ mod engine; mod server; use std::env; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, UdpSocket, SocketAddr}; use std::sync::Mutex; +use cadence::{QueuingMetricSink, UdpMetricSink}; use futures::{Future, Stream}; use hyper::Client; use hyper::server::Http; @@ -46,12 +48,22 @@ use tokio_core::reactor::Core; use self::server::Server; use self::engine::Engine; +fn init_metrics() -> QueuingMetricSink { + let socket = UdpSocket::bind("0.0.0.0:0").unwrap(); + socket.set_nonblocking(true).unwrap(); + let host = ("127.0.0.1", 8125); + let sink = UdpMetricSink::from(host, socket).unwrap(); + QueuingMetricSink::from(sink) +} + fn main() { let logger = slog::Logger::root( Mutex::new(slog_json::Json::default(std::io::stderr())).map(slog::Fuse), o!("version" => env!("CARGO_PKG_VERSION")) ); + let metrics = init_metrics(); + let mut core = Core::new() .expect("failed to create event loop"); @@ -71,7 +83,8 @@ fn main() { let http = Http::new(); - let engine = Engine::new(client.clone(), logger.clone()); + let mut engine = Engine::new(client.clone(), logger.clone()); + engine.set_metrics(metrics); let server = Server::new(logger.clone(), engine);