From 3ca746fe8670611a3044f7109dae3ef7e38a4789 Mon Sep 17 00:00:00 2001 From: Alexander Krantz Date: Mon, 16 May 2022 00:56:37 -0700 Subject: [PATCH] Better nested services (#15) --- Cargo.lock | 23 ++++++++++ Cargo.toml | 1 + docker/scripts/wafflemaker.hcl | 2 +- src/management/services.rs | 31 ++++++------- src/processor/jobs/delete_service.rs | 16 +++---- src/processor/jobs/update_service.rs | 20 ++++----- src/service/mod.rs | 12 +++-- src/service/name.rs | 65 ++++++++++++++++++++++++++++ src/service/registry.rs | 2 +- src/vault/models.rs | 2 +- src/vault/renewal.rs | 2 +- src/webhooks/handlers.rs | 2 +- wafflemaker.example.toml | 2 +- 13 files changed, 134 insertions(+), 46 deletions(-) create mode 100644 src/service/name.rs diff --git a/Cargo.lock b/Cargo.lock index 233b5aa..e117fc6 100755 --- a/Cargo.lock +++ b/Cargo.lock @@ -912,6 +912,15 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" +[[package]] +name = "itertools" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.9.0" @@ -1895,6 +1904,19 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shrinkwraprs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e63e6744142336dfb606fe2b068afa2e1cca1ee6a5d8377277a92945d81fa331" +dependencies = [ + "bitflags", + "itertools 0.8.2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "signal-hook-registry" version = "1.3.0" @@ -2441,6 +2463,7 @@ dependencies = [ "serde", "serde_json", "serde_with", + "shrinkwraprs", "sled", "structopt", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index d19f3e1..23b5c43 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ itertools = '0.10' once_cell = "1.8" serde = { version = "1.0", features = ["derive"] } serde_with = "1.9" +shrinkwraprs = "0.3" tokio = { version = "1.6", features = ["fs", "macros", "process", "rt", "rt-multi-thread", "signal", "time"] } tokio-stream = { version = "0.1", features = ["fs"] } diff --git a/docker/scripts/wafflemaker.hcl b/docker/scripts/wafflemaker.hcl index f00b54d..2a4e75b 100644 --- a/docker/scripts/wafflemaker.hcl +++ b/docker/scripts/wafflemaker.hcl @@ -1,5 +1,5 @@ # Allow reading and writing to services configuration KV -path "services/data/+" { +path "services/data/*" { capabilities = ["create", "update", "read"] } diff --git a/src/management/services.rs b/src/management/services.rs index 572489a..4b01faf 100644 --- a/src/management/services.rs +++ b/src/management/services.rs @@ -5,7 +5,7 @@ use crate::{ registry::REGISTRY, }; use serde::Serialize; -use warp::{http::StatusCode, Filter, Rejection, Reply}; +use warp::{http::StatusCode, path::Tail, Filter, Rejection, Reply}; /// Build the routes for services pub fn routes() -> impl Filter + Clone { @@ -15,18 +15,15 @@ pub fn routes() -> impl Filter + Clo .with(named_trace("list")); let get = warp::get() - .and(warp::path::param()) - .and(warp::path::end()) + .and(warp::path::tail()) .and_then(get) .with(named_trace("get")); let redeploy = warp::put() - .and(warp::path::param()) - .and(warp::path::end()) + .and(warp::path::tail()) .and_then(redeploy) .with(named_trace("redeploy")); let delete = warp::delete() - .and(warp::path::param()) - .and(warp::path::end()) + .and(warp::path::tail()) .and_then(delete) .with(named_trace("delete")); @@ -57,11 +54,13 @@ struct DependenciesResponse { } /// Get the configuration for a service -async fn get(service: String) -> Result { +async fn get(service: Tail) -> Result { + let service = service.as_str(); + let reg = REGISTRY.read().await; - let cfg = reg.get(&service).ok_or_else(warp::reject::not_found)?; + let cfg = reg.get(service).ok_or_else(warp::reject::not_found)?; - let deployment_id = deployer::instance().service_id(&service).await?; + let deployment_id = deployer::instance().service_id(service).await?; let dependencies = DependenciesResponse { postgres: cfg.dependencies.postgres("").is_some(), @@ -92,17 +91,19 @@ async fn get(service: String) -> Result { } /// Re-deploy a service -async fn redeploy(service: String) -> Result { +async fn redeploy(service: Tail) -> Result { + let service = service.as_str(); + let reg = REGISTRY.read().await; - let config = reg.get(&service).ok_or_else(warp::reject::not_found)?; + let config = reg.get(service).ok_or_else(warp::reject::not_found)?; - jobs::dispatch(UpdateService::new(config.clone(), service)); + jobs::dispatch(UpdateService::new(config.clone(), service.into())); Ok(StatusCode::NO_CONTENT) } /// Delete a service -async fn delete(service: String) -> Result { - jobs::dispatch(DeleteService::new(service)); +async fn delete(service: Tail) -> Result { + jobs::dispatch(DeleteService::new(service.as_str().into())); Ok(StatusCode::NO_CONTENT) } diff --git a/src/processor/jobs/delete_service.rs b/src/processor/jobs/delete_service.rs index e171c8c..79560f2 100644 --- a/src/processor/jobs/delete_service.rs +++ b/src/processor/jobs/delete_service.rs @@ -2,21 +2,20 @@ use super::Job; use crate::{ deployer, dns, fail_notify, notifier::{self, Event, State}, - service::registry::REGISTRY, + service::{registry::REGISTRY, ServiceName}, vault, }; use async_trait::async_trait; -use itertools::Itertools; use tracing::{debug, info, instrument}; #[derive(Debug)] pub struct DeleteService { - name: String, + name: ServiceName, } impl DeleteService { /// Create a new delete service job - pub fn new(name: String) -> Self { + pub fn new(name: ServiceName) -> Self { Self { name } } } @@ -31,11 +30,8 @@ impl Job for DeleteService { }; } - let sanitized_name = self.name.replace('/', "."); - let domain_name = self.name.split('/').rev().join("."); - let mut reg = REGISTRY.write().await; - if reg.remove(&self.name).is_none() { + if reg.remove(&self.name.proper).is_none() { info!("service was never deployed, skipping"); notifier::notify(Event::service_delete(&self.name, State::Success)).await; return; @@ -59,10 +55,10 @@ impl Job for DeleteService { fail!(vault::instance().revoke_leases(&id).await); - fail!(dns::instance().unregister(&domain_name).await); + fail!(dns::instance().unregister(&self.name.domain).await); if vault::instance() - .delete_database_role(&sanitized_name) + .delete_database_role(&self.name.sanitized) .await .is_err() { diff --git a/src/processor/jobs/update_service.rs b/src/processor/jobs/update_service.rs index 2dc9641..7cc7612 100644 --- a/src/processor/jobs/update_service.rs +++ b/src/processor/jobs/update_service.rs @@ -4,11 +4,10 @@ use crate::{ deployer::{self, CreateOpts}, dns, fail_notify, notifier::{self, Event, State}, - service::{registry::REGISTRY, AWSPart, Format, Secret, Service}, + service::{registry::REGISTRY, AWSPart, Format, Secret, Service, ServiceName}, vault::{self, Aws}, }; use async_trait::async_trait; -use itertools::Itertools; use rand::{distributions::Alphanumeric, Rng, RngCore, SeedableRng}; use rand_chacha::ChaCha20Rng; use tracing::{debug, error, info, instrument, warn}; @@ -16,12 +15,12 @@ use tracing::{debug, error, info, instrument, warn}; #[derive(Debug)] pub struct UpdateService { config: Service, - name: String, + name: ServiceName, } impl UpdateService { /// Create a new update service job - pub fn new(config: Service, name: String) -> Self { + pub fn new(config: Service, name: ServiceName) -> Self { Self { config, name } } } @@ -39,9 +38,6 @@ impl Job for UpdateService { let config = config::instance(); let service = &self.config; - let sanitized_name = self.name.replace('/', "."); - let domain_name = self.name.split('/').rev().join("."); - notifier::notify(Event::service_update(&self.name, State::InProgress)).await; // Update the service in the registry @@ -50,13 +46,13 @@ impl Job for UpdateService { // Create the base container creation args let mut options = CreateOpts::builder() - .name(&self.name) + .name(&*self.name) .image(&service.docker.image, &service.docker.tag); if service.web.enabled { let domain = match service.web.domain.clone() { Some(d) => d, - None => format!("{}.{}", &domain_name, &config.deployment.domain), + None => format!("{}.{}", &self.name.domain, &config.deployment.domain), }; options = options.routing(domain, service.web.path.as_deref()); @@ -120,7 +116,7 @@ impl Job for UpdateService { } info!("loaded secrets from vault into environment"); - if let Some(postgres) = service.dependencies.postgres(&sanitized_name) { + if let Some(postgres) = service.dependencies.postgres(&self.name.sanitized) { // Create the role if it doesn't exist let roles = fail!(vault::instance().list_database_roles().await); if !roles.contains(&postgres.role.to_owned()) { @@ -150,7 +146,7 @@ impl Job for UpdateService { info!("loaded service dependencies into environment"); let known_services = fail!(deployer::instance().list().await); - let previous_id = known_services.get(&self.name); + let previous_id = known_services.get(&*self.name); // Perform a rolling update of the service (if a previous version existed) // Flow (assuming previous version existed): @@ -204,7 +200,7 @@ impl Job for UpdateService { // Register the internal DNS record(s) let ip = fail!(deployer::instance().ip(&new_id).await); - fail!(dns::instance().register(&domain_name, &ip).await); + fail!(dns::instance().register(&self.name.domain, &ip).await); info!("deployed with id \"{}\"", new_id); notifier::notify(Event::service_update(&self.name, State::Success)).await; diff --git a/src/service/mod.rs b/src/service/mod.rs index c2e2b42..7a9514b 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -2,14 +2,17 @@ use crate::config; use globset::{Glob, GlobSet, GlobSetBuilder}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr, NoneAsEmptyString}; +use std::fmt::Debug; use std::{collections::HashMap, ffi::OsStr, path::Path}; use tokio::fs; mod dependency; +mod name; pub mod registry; mod secret; use dependency::*; +pub use name::ServiceName; pub use secret::{Format, Part as AWSPart, Secret}; /// The configuration for a service @@ -34,15 +37,18 @@ impl Service { } /// Generate the name of a service from its file path - pub fn name(path: &Path) -> String { - path.strip_prefix(&config::instance().git.clone_to) + pub fn name(path: &Path) -> ServiceName { + let name = path + .strip_prefix(&config::instance().git.clone_to) .unwrap_or(path) .with_extension("") .iter() .map(OsStr::to_str) .map(Option::unwrap) .collect::>() - .join("/") + .join("/"); + + ServiceName::new(name) } } diff --git a/src/service/name.rs b/src/service/name.rs new file mode 100644 index 0000000..bb9bf82 --- /dev/null +++ b/src/service/name.rs @@ -0,0 +1,65 @@ +use itertools::Itertools; +use shrinkwraprs::Shrinkwrap; +use std::fmt::{Debug, Display, Formatter}; + +/// The different ways of referring to a service. Can be used as a standard string via [shrinkwraprs::Shrinkwrap] +#[derive(Shrinkwrap)] +pub struct ServiceName { + /// The proper name of the service by which most things should refer to it + #[shrinkwrap(main_field)] + pub proper: String, + /// The domain/subdomain name of the service + pub domain: String, + /// A sanitized version of the name containing only a-z, A-Z, 0-9, -, . + pub sanitized: String, +} + +impl ServiceName { + pub(super) fn new>(name: S) -> ServiceName { + let proper = name.into(); + let domain = proper.split('/').rev().join("."); + let sanitized = proper.replace('/', "."); + + ServiceName { + proper, + domain, + sanitized, + } + } +} + +impl AsRef for ServiceName { + fn as_ref(&self) -> &str { + self.proper.as_ref() + } +} + +impl Debug for ServiceName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Debug::fmt(&self.proper, f) + } +} + +impl Display for ServiceName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Display::fmt(&self.proper, f) + } +} + +impl From<&str> for ServiceName { + fn from(name: &str) -> Self { + ServiceName::new(name) + } +} + +impl From for ServiceName { + fn from(name: String) -> Self { + ServiceName::new(name) + } +} + +impl From<&String> for ServiceName { + fn from(name: &String) -> Self { + ServiceName::new(name) + } +} diff --git a/src/service/registry.rs b/src/service/registry.rs index f827356..4db2377 100644 --- a/src/service/registry.rs +++ b/src/service/registry.rs @@ -42,7 +42,7 @@ async fn load_dir(reg: &mut HashMap, path: &Path) -> Result<()> let service = Service::parse(&entry.path()).await?; debug!("loaded service {}", &name); - reg.insert(name, service); + reg.insert(name.proper, service); } Ok(()) diff --git a/src/vault/models.rs b/src/vault/models.rs index 66fcddd..d3b2cf9 100644 --- a/src/vault/models.rs +++ b/src/vault/models.rs @@ -60,7 +60,7 @@ impl<'paths> Capabilities<'paths> { m.insert("aws/creds/+", set![Read]); m.insert("database/creds/+", set![Read]); m.insert("database/roles/+", set![List, Create, Delete]); - m.insert("services/data/+", set![Create, Read, Update]); + m.insert("services/data/*", set![Create, Read, Update]); m } diff --git a/src/vault/renewal.rs b/src/vault/renewal.rs index 5d4d739..7c277d8 100644 --- a/src/vault/renewal.rs +++ b/src/vault/renewal.rs @@ -70,7 +70,7 @@ pub async fn leases(interval: Duration, max_percent: f64, mut stop: Receiver<()> Err(e) => { let reg = REGISTRY.read().await; if let Some(config) = reg.get(service) { - jobs::dispatch(UpdateService::new(config.clone(), service.to_owned())); + jobs::dispatch(UpdateService::new(config.clone(), service.into())); } warn!(parent: &span, id = %lease.id, error = %e, "failed to renew lease"); diff --git a/src/webhooks/handlers.rs b/src/webhooks/handlers.rs index 0818572..75c6d48 100755 --- a/src/webhooks/handlers.rs +++ b/src/webhooks/handlers.rs @@ -46,7 +46,7 @@ pub async fn docker(body: Docker, authorization: String) -> Result