instrument analyze timings

Cargo.lock

@@ -68,6 +68,14 @@ dependencies = [
  "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "cadence"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "crossbeam 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.4"
@@ -104,6 +112,11 @@ dependencies = [
  "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "crossbeam"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "dtoa"
 version = "0.4.2"
@@ -682,6 +695,7 @@ name = "shiny-robots"
 version = "0.1.0"
 dependencies = [
  "badge 0.2.0",
+ "cadence 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper 0.11.18 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -947,11 +961,13 @@ dependencies = [
 "checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf"
 "checksum byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "652805b7e73fada9d85e9a6682a4abd490cb52d96aeecc12e33a0de34dfd0d23"
 "checksum bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1b7db437d718977f6dc9b2e3fd6fc343c02ac6b899b73fdd2179163447bd9ce9"
+"checksum cadence 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f1725ad9d5c8251726f1d7207d49e635e92448010fca03bddc7f07f0a4488a33"
 "checksum cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "deaf9ec656256bb25b404c51ef50097207b9cbb29c933d31f92cae5a8a0ffee0"
 "checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de"
 "checksum chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c20ebe0b2b08b0aeddba49c609fe7957ba2e33449882cb186a180bc60682fa9"
 "checksum core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25bfd746d203017f7d5cbd31ee5d8e17f94b6521c7af77ece6c9e4b2d4b16c67"
 "checksum core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "065a5d7ffdcbc8fa145d6f0746f3555025b9097a9e9cda59f7467abae670c78d"
+"checksum crossbeam 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "bd66663db5a988098a89599d4857919b3acf7f61402e61365acfd3919857b9be"
 "checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab"
 "checksum failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "934799b6c1de475a012a02dab0ace1ace43789ee4b99bcfbf1a2e3e8ced5de82"
 "checksum failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c7cdda555bb90c9bb67a3b670a0f42de8e73f5981524123ad8578aafec8ddb8b"

Cargo.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 authors = ["Sam Rijs <srijs@airpost.net>"]
 
 [dependencies]
+cadence = "0.13.1"
 failure = "0.1.1"
 futures = "0.1.18"
 hyper = "0.11.15"

src/engine/mod.rs

@@ -2,6 +2,8 @@ use std::collections::HashSet;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
+use cadence::prelude::*;
+use cadence::{MetricSink, NopMetricSink, StatsdClient};
 use failure::Error;
 use futures::{Future, future};
 use futures::future::join_all;
@@ -34,6 +36,7 @@ type HttpClient = Client<HttpsConnector<HttpConnector>>;
 pub struct Engine {
     client: HttpClient,
     logger: Logger,
+    metrics: StatsdClient,
 
     query_crate: Arc<Cache<QueryCrate<HttpClient>>>,
     get_popular_crates: Arc<Cache<GetPopularCrates<HttpClient>>>,
@@ -43,12 +46,14 @@ pub struct Engine {
 
 impl Engine {
     pub fn new(client: Client<HttpsConnector<HttpConnector>>, logger: Logger) -> Engine {
+        let metrics = StatsdClient::from_sink("engine", NopMetricSink);
+
         let query_crate = Cache::new(QueryCrate(client.clone()), Duration::from_secs(300), 500);
         let get_popular_crates = Cache::new(GetPopularCrates(client.clone()), Duration::from_secs(10), 1);
         let get_popular_repos = Cache::new(GetPopularRepos(client.clone()), Duration::from_secs(10), 1);
 
         Engine {
-            client: client.clone(), logger,
+            client: client.clone(), logger, metrics,
 
             query_crate: Arc::new(query_crate),
             get_popular_crates: Arc::new(get_popular_crates),
@@ -56,6 +61,10 @@ impl Engine {
             retrieve_file_at_path: Arc::new(RetrieveFileAtPath(client))
         }
     }
+
+    pub fn set_metrics<M: MetricSink + Send + Sync + 'static>(&mut self, sink: M) {
+        self.metrics = StatsdClient::from_sink("engine", sink);
+    }
 }
 
 pub struct AnalyzeDependenciesOutcome {
@@ -100,22 +109,28 @@ impl Engine {
         let start = Instant::now();
 
         let entry_point = RelativePath::new("/").to_relative_path_buf();
-        let manifest_future = CrawlManifestFuture::new(self, repo_path, entry_point);
+        let manifest_future = CrawlManifestFuture::new(self, repo_path.clone(), entry_point);
 
         let engine = self.clone();
         manifest_future.and_then(move |manifest_output| {
+            let engine_for_analyze = engine.clone();
             let futures = manifest_output.crates.into_iter().map(move |(crate_name, deps)| {
-                let analyzed_deps_future = AnalyzeDependenciesFuture::new(&engine, deps);
+                let analyzed_deps_future = AnalyzeDependenciesFuture::new(&engine_for_analyze, deps);
 
                 analyzed_deps_future.map(move |analyzed_deps| (crate_name, analyzed_deps))
             });
 
-            join_all(futures).map(move |crates| {
+            join_all(futures).and_then(move |crates| {
                 let duration = start.elapsed();
+                engine.metrics.time_duration_with_tags("analyze_duration", duration)
+                    .with_tag("repo_site", repo_path.site.as_ref())
+                    .with_tag("repo_qual", repo_path.qual.as_ref())
+                    .with_tag("repo_name", repo_path.name.as_ref())
+                    .send()?;
 
-                AnalyzeDependenciesOutcome {
+                Ok(AnalyzeDependenciesOutcome {
                     crates, duration
-                }
+                })
             })
         })
     }

src/main.rs

@@ -4,6 +4,7 @@
 #![feature(proc_macro)]
 
 extern crate badge;
+extern crate cadence;
 #[macro_use] extern crate failure;
 #[macro_use] extern crate futures;
 extern crate hyper;
@@ -33,9 +34,10 @@ mod engine;
 mod server;
 
 use std::env;
-use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::net::{IpAddr, Ipv4Addr, UdpSocket, SocketAddr};
 use std::sync::Mutex;
 
+use cadence::{QueuingMetricSink, UdpMetricSink};
 use futures::{Future, Stream};
 use hyper::Client;
 use hyper::server::Http;
@@ -46,12 +48,22 @@ use tokio_core::reactor::Core;
 use self::server::Server;
 use self::engine::Engine;
 
+fn init_metrics() -> QueuingMetricSink {
+    let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+    socket.set_nonblocking(true).unwrap();
+    let host = ("127.0.0.1", 8125);
+    let sink = UdpMetricSink::from(host, socket).unwrap();
+    QueuingMetricSink::from(sink)
+}
+
 fn main() {
     let logger = slog::Logger::root(
         Mutex::new(slog_json::Json::default(std::io::stderr())).map(slog::Fuse),
         o!("version" => env!("CARGO_PKG_VERSION"))
     );
 
+    let metrics = init_metrics();
+
     let mut core = Core::new()
         .expect("failed to create event loop");
 
@@ -71,7 +83,8 @@ fn main() {
 
     let http = Http::new();
 
-    let engine = Engine::new(client.clone(), logger.clone());
+    let mut engine = Engine::new(client.clone(), logger.clone());
+    engine.set_metrics(metrics);
 
     let server = Server::new(logger.clone(), engine);