Skip to content

Commit

Permalink
Faucet Filtering (#19988)
Browse files Browse the repository at this point in the history
## Description 

Faucet filtering for paths /v1/gas, /gas, and group all /v1/status
More conditional checks to prevent panic

## Test plan 

How did you test the new or updated feature?

Tested locally, to confirm on testnet

## Release notes

Check each box that your changes affect. If none of the boxes relate to
your changes, release notes aren't required.

For each box you select, include information after the relevant heading
that describes the impact of your changes that a user might notice and
any actions they must take to implement updates.

- [ ] Protocol: 
- [ ] Nodes (Validators and Full nodes): 
- [ ] Indexer: 
- [ ] JSON-RPC: 
- [ ] GraphQL: 
- [ ] CLI: 
- [ ] Rust SDK:
- [ ] REST API:
  • Loading branch information
leecchh authored Oct 24, 2024
1 parent 82ab06a commit eef06e5
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 39 deletions.
52 changes: 44 additions & 8 deletions crates/sui-faucet/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -135,47 +135,83 @@ impl FaucetMetrics {

impl MetricsCallbackProvider for RequestMetrics {
fn on_request(&self, path: String) {
let normalized_path = normalize_path(&path);
if !is_path_tracked(normalized_path) {
return;
}

self.total_requests_received
.with_label_values(&[path.as_str()])
.with_label_values(&[normalized_path])
.inc();
}

fn on_response(&self, path: String, latency: Duration, _status: u16, grpc_status_code: Code) {
let normalized_path = normalize_path(&path);
if !is_path_tracked(normalized_path) {
return;
}

self.process_latency
.with_label_values(&[path.as_str()])
.with_label_values(&[normalized_path])
.observe(latency.as_secs_f64());

match grpc_status_code {
Code::Ok => {
self.total_requests_succeeded
.with_label_values(&[path.as_str()])
.with_label_values(&[normalized_path])
.inc();
}
Code::Unavailable | Code::ResourceExhausted => {
self.total_requests_shed
.with_label_values(&[path.as_str()])
.with_label_values(&[normalized_path])
.inc();
}
_ => {
self.total_requests_failed
.with_label_values(&[path.as_str()])
.with_label_values(&[normalized_path])
.inc();
}
}
}

fn on_start(&self, path: &str) {
let normalized_path = normalize_path(path);
if !is_path_tracked(normalized_path) {
return;
}

self.current_requests_in_flight
.with_label_values(&[path])
.with_label_values(&[normalized_path])
.inc();
}

fn on_drop(&self, path: &str) {
let normalized_path = normalize_path(path);
if !is_path_tracked(normalized_path) {
return;
}

self.total_requests_disconnected
.with_label_values(&[path])
.with_label_values(&[normalized_path])
.inc();
self.current_requests_in_flight
.with_label_values(&[path])
.with_label_values(&[normalized_path])
.dec();
}
}

/// Normalizes the given path to handle variations across different deployments.
/// Specifically, it trims dynamic segments from the `/v1/status/` endpoint.
pub fn normalize_path(path: &str) -> &str {
if path.starts_with("/v1/status/") {
return "/v1/status";
}

path
}

/// Determines whether the given path should be tracked for metrics collection.
/// Only specified paths relevant to monitoring are included.
pub fn is_path_tracked(path: &str) -> bool {
matches!(path, "/v1/gas" | "/gas" | "/v1/status")
}
77 changes: 46 additions & 31 deletions crates/sui-faucet/src/metrics_layer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use prometheus::{HistogramTimer, Registry};
use tower::{load_shed::error::Overloaded, BoxError, Layer, Service, ServiceExt};
use tracing::{error, info, warn};

use crate::metrics::RequestMetrics;
use crate::metrics::{is_path_tracked, normalize_path, RequestMetrics};
use http::Request;

/// Tower Layer for tracking metrics in Prometheus related to number, success-rate and latency of
Expand Down Expand Up @@ -81,16 +81,19 @@ where

let future = Box::pin(async move {
let resp = inner.oneshot(req).await;
match &resp {
Ok(resp) if !resp.status().is_success() => {
metrics.failed(None, Some(resp.status()))
}
Ok(_) => metrics.succeeded(),
Err(err) => {
if err.is::<Overloaded>() {
metrics.shed();
} else {
metrics.failed(Some(err), None);

if let Some(metrics) = metrics {
match &resp {
Ok(resp) if !resp.status().is_success() => {
metrics.failed(None, Some(resp.status()))
}
Ok(_) => metrics.succeeded(),
Err(err) => {
if err.is::<Overloaded>() {
metrics.shed();
} else {
metrics.failed(Some(err), None);
}
}
}
}
Expand All @@ -110,25 +113,31 @@ impl<Res> Future for RequestMetricsFuture<Res> {
}

impl MetricsGuard {
fn new(metrics: Arc<RequestMetrics>, path: &str) -> Self {
fn new(metrics: Arc<RequestMetrics>, path: &str) -> Option<Self> {
let normalized_path = normalize_path(path);

if !is_path_tracked(normalized_path) {
return None;
}

metrics
.total_requests_received
.with_label_values(&[path])
.with_label_values(&[normalized_path])
.inc();
metrics
.current_requests_in_flight
.with_label_values(&[path])
.with_label_values(&[normalized_path])
.inc();
MetricsGuard {
Some(MetricsGuard {
timer: Some(
metrics
.process_latency
.with_label_values(&[path])
.with_label_values(&[normalized_path])
.start_timer(),
),
metrics,
path: path.to_string(),
}
path: normalized_path.to_string(),
})
}

fn succeeded(mut self) {
Expand Down Expand Up @@ -183,22 +192,28 @@ impl MetricsGuard {

impl Drop for MetricsGuard {
fn drop(&mut self) {
self.metrics
if self
.metrics
.current_requests_in_flight
.with_label_values(&[&self.path])
.dec();

// Request was still in flight when the guard was dropped, implying the client disconnected.
if let Some(timer) = self.timer.take() {
let elapsed = timer.stop_and_record();
.get_metric_with_label_values(&[&self.path])
.is_ok()
{
self.metrics
.total_requests_disconnected
.current_requests_in_flight
.with_label_values(&[&self.path])
.inc();
info!(
"Request disconnected for path {} in {:.2}s",
self.path, elapsed
);
.dec();

if let Some(timer) = self.timer.take() {
let elapsed = timer.stop_and_record();
self.metrics
.total_requests_disconnected
.with_label_values(&[&self.path])
.inc();
info!(
"Request disconnected for path {} in {:.2}s",
self.path, elapsed
);
}
}
}
}

0 comments on commit eef06e5

Please sign in to comment.