From 7527a71212b182c18f0d5c38fd5511cdd0fad1b5 Mon Sep 17 00:00:00 2001 From: Nate Anderson Date: Tue, 2 Apr 2024 11:41:18 -1000 Subject: [PATCH] Copy the average metrics back to sum. Add progress bars/indicators for the different steps. --- Cargo.lock | 39 +++++++++++++++++++ momento/Cargo.toml | 1 + momento/src/commands/cloud_linter/dynamodb.rs | 26 +++++++++---- .../src/commands/cloud_linter/elasticache.rs | 12 +++--- .../src/commands/cloud_linter/linter_cli.rs | 2 + momento/src/commands/cloud_linter/metrics.rs | 11 ++++-- 6 files changed, 74 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 019e4fc..c3f7582 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -772,6 +772,19 @@ version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5458d9d1a587efaf5091602c59d299696a3877a439c8f6d461a2d3cce11df87a" +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.52.0", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -925,6 +938,12 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.32" @@ -1442,6 +1461,19 @@ dependencies = [ "hashbrown 0.14.3", ] +[[package]] +name = "indicatif" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "instant" version = "0.1.12" @@ -1661,6 +1693,7 @@ dependencies = [ "governor", "home", "humantime", + "indicatif", "lazy_static", "log", "momento", @@ -1809,6 +1842,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "objc" version = "0.2.7" diff --git a/momento/Cargo.toml b/momento/Cargo.toml index 7213709..12dabae 100644 --- a/momento/Cargo.toml +++ b/momento/Cargo.toml @@ -24,6 +24,7 @@ aws-sdk-cloudwatch = "1.19.0" aws-sdk-dynamodb = "1.19.0" aws-sdk-elasticache = "1.18.0" phf = { version = "0.11", features = ["macros"] } +indicatif = "0.17.8" [dev-dependencies] assert_cmd = "2.0.2" diff --git a/momento/src/commands/cloud_linter/dynamodb.rs b/momento/src/commands/cloud_linter/dynamodb.rs index 9332be9..69d555a 100644 --- a/momento/src/commands/cloud_linter/dynamodb.rs +++ b/momento/src/commands/cloud_linter/dynamodb.rs @@ -1,9 +1,11 @@ use std::collections::HashMap; use std::sync::Arc; +use std::time::Duration; use aws_config::SdkConfig; use aws_sdk_dynamodb::types::{TimeToLiveDescription, TimeToLiveStatus}; use governor::DefaultDirectRateLimiter; +use indicatif::{ProgressBar, ProgressStyle}; use phf::{phf_map, Map}; use serde::{Deserialize, Serialize}; @@ -11,10 +13,13 @@ use crate::commands::cloud_linter::metrics::{Metric, MetricTarget, ResourceWithM use crate::commands::cloud_linter::resource::{DynamoDbResource, Resource, ResourceType}; use crate::commands::cloud_linter::utils::rate_limit; use crate::error::CliError; -use crate::utils::console::console_info; const DDB_TABLE_METRICS: Map<&'static str, &'static [&'static str]> = phf_map! { "Sum" => &[ + "ConsumedReadCapacityUnits", + "ConsumedWriteCapacityUnits", + "ProvisionedReadCapacityUnits", + "ProvisionedWriteCapacityUnits", "ReadThrottleEvents", "WriteThrottleEvents", "TimeToLiveDeletedItemCount", @@ -39,6 +44,10 @@ const DDB_TABLE_METRICS: Map<&'static str, &'static [&'static str]> = phf_map! { const DDB_GSI_METRICS: Map<&'static str, &'static [&'static str]> = phf_map! { "Sum" => &[ + "ConsumedReadCapacityUnits", + "ConsumedWriteCapacityUnits", + "ProvisionedReadCapacityUnits", + "ProvisionedWriteCapacityUnits", "ReadThrottleEvents", "WriteThrottleEvents", ], @@ -140,10 +149,7 @@ impl ResourceWithMetrics for DynamoDbResource { targets: DDB_GSI_METRICS, }) } - ResourceType::ElastiCacheRedisNode => Err(CliError { - msg: "Invalid resource type".to_string(), - }), - ResourceType::ElastiCacheMemcachedNode => Err(CliError { + _ => Err(CliError { msg: "Invalid resource type".to_string(), }), } @@ -164,10 +170,14 @@ pub(crate) async fn get_ddb_resources( ) -> Result, CliError> { let ddb_client = aws_sdk_dynamodb::Client::new(&config); - console_info!("Listing Dynamo DB tables"); + let bar = ProgressBar::new_spinner().with_message("Listing Dynamo DB tables"); + bar.enable_steady_tick(Duration::from_millis(100)); let table_names = list_table_names(&ddb_client, Arc::clone(&limiter)).await?; + bar.finish(); - console_info!("Describing tables"); + let bar = + ProgressBar::new(table_names.len() as u64).with_message("Describing Dynamo DB tables"); + bar.set_style(ProgressStyle::with_template(" {msg} {bar} {eta}").expect("invalid template")); let mut resources = Vec::new(); for table_name in table_names { let instances = fetch_ddb_resources(&ddb_client, &table_name, Arc::clone(&limiter)).await?; @@ -176,7 +186,9 @@ pub(crate) async fn get_ddb_resources( .map(Resource::DynamoDb) .collect::>(); resources.extend(wrapped_resources); + bar.inc(1); } + bar.finish(); Ok(resources) } diff --git a/momento/src/commands/cloud_linter/elasticache.rs b/momento/src/commands/cloud_linter/elasticache.rs index 4a31885..2e5c123 100644 --- a/momento/src/commands/cloud_linter/elasticache.rs +++ b/momento/src/commands/cloud_linter/elasticache.rs @@ -1,9 +1,11 @@ use std::collections::HashMap; use std::sync::Arc; +use std::time::Duration; use aws_config::SdkConfig; use aws_sdk_elasticache::types::CacheCluster; use governor::DefaultDirectRateLimiter; +use indicatif::ProgressBar; use phf::{phf_map, Map}; use serde::Serialize; @@ -11,7 +13,6 @@ use crate::commands::cloud_linter::metrics::{Metric, MetricTarget, ResourceWithM use crate::commands::cloud_linter::resource::{ElastiCacheResource, Resource, ResourceType}; use crate::commands::cloud_linter::utils::rate_limit; use crate::error::CliError; -use crate::utils::console::console_info; pub(crate) const CACHE_METRICS: Map<&'static str, &'static [&'static str]> = phf_map! { "Sum" => &[ @@ -84,10 +85,7 @@ impl ResourceWithMetrics for ElastiCacheResource { ]), targets: CACHE_METRICS, }), - ResourceType::DynamoDbGsi => Err(CliError { - msg: "Invalid resource type".to_string(), - }), - ResourceType::DynamoDbTable => Err(CliError { + _ => Err(CliError { msg: "Invalid resource type".to_string(), }), } @@ -106,7 +104,6 @@ pub(crate) async fn get_elasticache_resources( config: &SdkConfig, limiter: Arc, ) -> Result, CliError> { - console_info!("Describing ElastiCache clusters"); let region = config.region().map(|r| r.as_ref()).ok_or(CliError { msg: "No region configured for client".to_string(), })?; @@ -121,6 +118,8 @@ async fn describe_clusters( elasticache_client: &aws_sdk_elasticache::Client, limiter: Arc, ) -> Result, CliError> { + let bar = ProgressBar::new_spinner().with_message("Describing ElastiCache clusters"); + bar.enable_steady_tick(Duration::from_millis(100)); let mut elasticache_clusters = Vec::new(); let mut elasticache_stream = elasticache_client .describe_cache_clusters() @@ -142,6 +141,7 @@ async fn describe_clusters( } } } + bar.finish(); Ok(elasticache_clusters) } diff --git a/momento/src/commands/cloud_linter/linter_cli.rs b/momento/src/commands/cloud_linter/linter_cli.rs index f7906d7..396dc5c 100644 --- a/momento/src/commands/cloud_linter/linter_cli.rs +++ b/momento/src/commands/cloud_linter/linter_cli.rs @@ -21,9 +21,11 @@ pub async fn run_cloud_linter(region: String) -> Result<(), CliError> { let limiter = Arc::new(RateLimiter::direct(quota)); let mut resources = get_ddb_resources(&config, Arc::clone(&limiter)).await?; + let mut elasticache_resources = get_elasticache_resources(&config, Arc::clone(&limiter)).await?; resources.append(&mut elasticache_resources); + let resources = append_metrics_to_resources(&config, Arc::clone(&limiter), resources).await?; let data_format = DataFormat { resources }; diff --git a/momento/src/commands/cloud_linter/metrics.rs b/momento/src/commands/cloud_linter/metrics.rs index 70dc6ff..eda7b48 100644 --- a/momento/src/commands/cloud_linter/metrics.rs +++ b/momento/src/commands/cloud_linter/metrics.rs @@ -1,8 +1,6 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::commands::cloud_linter::resource::Resource; -use crate::commands::cloud_linter::utils::rate_limit; use aws_config::SdkConfig; use aws_sdk_cloudwatch::primitives::DateTime; use aws_sdk_cloudwatch::types::Metric as CloudwatchMetric; @@ -10,11 +8,13 @@ use aws_sdk_cloudwatch::types::{Dimension, MetricDataQuery, MetricStat}; use aws_sdk_cloudwatch::Client; use chrono::{Duration, Utc}; use governor::DefaultDirectRateLimiter; +use indicatif::{ProgressBar, ProgressStyle}; use phf::Map; use serde::{Deserialize, Serialize}; +use crate::commands::cloud_linter::resource::Resource; +use crate::commands::cloud_linter::utils::rate_limit; use crate::error::CliError; -use crate::utils::console::console_info; #[derive(Serialize, Deserialize)] pub(crate) struct Metric { @@ -68,7 +68,8 @@ pub(crate) async fn append_metrics_to_resources( limiter: Arc, mut resources: Vec, ) -> Result, CliError> { - console_info!("Getting metrics..."); + let bar = ProgressBar::new(resources.len() as u64).with_message("Querying metrics"); + bar.set_style(ProgressStyle::with_template(" {msg} {bar} {eta}").expect("invalid template")); let metrics_client = Client::new(config); for resource in &mut resources { @@ -84,7 +85,9 @@ pub(crate) async fn append_metrics_to_resources( .await?; } } + bar.inc(1); } + bar.finish(); Ok(resources) }