From bbe630e8222c88c3b95e6eed35ab784871e6935e Mon Sep 17 00:00:00 2001 From: kjuulh Date: Mon, 20 May 2024 23:05:20 +0200 Subject: [PATCH] feat: add variables Signed-off-by: kjuulh --- Cargo.lock | 29 ++++ crates/cuddle-clusters/Cargo.toml | 1 + crates/cuddle-clusters/src/catalog.rs | 1 + .../src/catalog/cuddle_vars.rs | 160 ++++++++++++++++++ crates/cuddle-clusters/src/components.rs | 75 ++++++++ crates/cuddle-clusters/src/lib.rs | 6 +- crates/cuddle-clusters/src/process.rs | 57 +++++-- crates/cuddle-clusters/tests/common.rs | 35 ++-- crates/cuddle-clusters/tests/cuddle_vars.rs | 49 ++++++ .../tests/cuddle_vars/basic/cuddle.yaml | 10 ++ crates/cuddle-clusters/tests/tests.rs | 18 +- .../tests/with_cuddle_vars/cuddle.yaml | 5 + .../with_cuddle_vars/expected/dev/some.yaml | 2 + .../templates/clusters/some.yaml.jinja2 | 3 + 14 files changed, 429 insertions(+), 22 deletions(-) create mode 100644 crates/cuddle-clusters/src/catalog.rs create mode 100644 crates/cuddle-clusters/src/catalog/cuddle_vars.rs create mode 100644 crates/cuddle-clusters/src/components.rs create mode 100644 crates/cuddle-clusters/tests/cuddle_vars.rs create mode 100644 crates/cuddle-clusters/tests/cuddle_vars/basic/cuddle.yaml create mode 100644 crates/cuddle-clusters/tests/with_cuddle_vars/cuddle.yaml create mode 100644 crates/cuddle-clusters/tests/with_cuddle_vars/expected/dev/some.yaml create mode 100644 crates/cuddle-clusters/tests/with_cuddle_vars/templates/clusters/some.yaml.jinja2 diff --git a/Cargo.lock b/Cargo.lock index 1d05f13..8655be7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -388,6 +388,7 @@ dependencies = [ "axum", "clap", "dotenv", + "futures", "minijinja", "serde", "serde_yaml", @@ -533,6 +534,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -577,6 +593,17 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.65", +] + [[package]] name = "futures-sink" version = "0.3.30" @@ -595,8 +622,10 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", diff --git a/crates/cuddle-clusters/Cargo.toml b/crates/cuddle-clusters/Cargo.toml index cfb6bc2..a87623c 100644 --- a/crates/cuddle-clusters/Cargo.toml +++ b/crates/cuddle-clusters/Cargo.toml @@ -27,6 +27,7 @@ serde_yaml = "0.9.34" tokio-stream = "0.1.15" walkdir = "2.5.0" minijinja = "2.0.1" +futures = "0.3.30" [[test]] name = "integration" diff --git a/crates/cuddle-clusters/src/catalog.rs b/crates/cuddle-clusters/src/catalog.rs new file mode 100644 index 0000000..e6ccdfb --- /dev/null +++ b/crates/cuddle-clusters/src/catalog.rs @@ -0,0 +1 @@ +pub mod cuddle_vars; diff --git a/crates/cuddle-clusters/src/catalog/cuddle_vars.rs b/crates/cuddle-clusters/src/catalog/cuddle_vars.rs new file mode 100644 index 0000000..9cf3206 --- /dev/null +++ b/crates/cuddle-clusters/src/catalog/cuddle_vars.rs @@ -0,0 +1,160 @@ +use std::{collections::HashMap, ops::Deref, path::Path}; + +use anyhow::Context; +use futures::{future::BoxFuture, FutureExt}; + +use crate::Component; + +#[derive(PartialEq, Eq, Debug, Clone)] +pub enum CuddleVariable { + Object(Box), + Array(Vec), + String(String), +} + +impl TryFrom for CuddleVariable { + type Error = anyhow::Error; + + fn try_from(value: serde_yaml::Value) -> Result { + match value { + serde_yaml::Value::Sequence(sequence) => { + let mut items = Vec::new(); + + for item in sequence { + items.push(item.try_into()?) + } + + Ok(Self::Array(items)) + } + serde_yaml::Value::Mapping(mapping) => { + let obj: CuddleVariables = mapping.try_into()?; + + Ok(Self::Object(Box::new(obj))) + } + serde_yaml::Value::String(s) => Ok(Self::String(s)), + serde_yaml::Value::Number(num) => Ok(Self::String(num.to_string())), + serde_yaml::Value::Bool(bool) => Ok(Self::String(bool.to_string())), + _ => Err(anyhow::anyhow!("cannot handle type of serde value")), + } + } +} + +#[derive(PartialEq, Eq, Debug, Clone)] +pub struct CuddleVariables(pub HashMap); + +impl TryFrom for CuddleVariables { + type Error = anyhow::Error; + + fn try_from(value: serde_yaml::Mapping) -> Result { + let mut variables = CuddleVariables(HashMap::default()); + + for (k, v) in value { + let var: CuddleVariable = v.try_into()?; + + variables.0.insert( + k.as_str() + .ok_or(anyhow::anyhow!("key cannot be anything else than a string"))? + .to_string(), + var, + ); + } + + Ok(variables) + } +} + +impl TryFrom for CuddleVariables { + type Error = anyhow::Error; + + fn try_from(value: serde_yaml::Value) -> Result { + match value { + serde_yaml::Value::Null => anyhow::bail!("cannot handle null"), + serde_yaml::Value::Bool(_) => anyhow::bail!("cannot handle bool"), + serde_yaml::Value::Number(_) => anyhow::bail!("cannot handle number"), + serde_yaml::Value::String(_) => anyhow::bail!("cannot handle string"), + serde_yaml::Value::Sequence(_) => anyhow::bail!("cannot handle sequence"), + serde_yaml::Value::Mapping(mapping) => mapping.try_into(), + serde_yaml::Value::Tagged(_) => anyhow::bail!("cannot handle tagged"), + } + } +} + +#[derive(PartialEq, Eq, Debug, Clone)] +pub struct CuddleVars { + pub variables: CuddleVariables, +} + +const PARENT_PLAN_PREFIX: &str = ".cuddle/plan"; +const CUDDLE_FILE: &str = "cuddle.yaml"; + +impl CuddleVars { + pub async fn new(path: &Path) -> anyhow::Result { + let variables = load_cuddle_file(path).await?; + + Ok(Self { variables }) + } +} + +fn load_cuddle_file(path: &Path) -> BoxFuture<'static, anyhow::Result> { + let path = path.to_path_buf(); + + async move { + let cuddle_file = path.join(CUDDLE_FILE); + let cuddle_file_content = tokio::fs::read(cuddle_file) + .await + .context(format!("failed to read cuddle file: {}", path.display()))?; + let cuddle_serde: serde_yaml::Value = serde_yaml::from_slice(&cuddle_file_content)?; + match cuddle_serde.get("vars") { + Some(vars) => { + let parent_file = path.join(PARENT_PLAN_PREFIX); + if parent_file.exists() { + // FIXME: Merge parent map + //let parent_plan = load_cuddle_file(&parent_file).await?; + //Err(anyhow::anyhow!("not implemented yet")) + + Ok(CuddleVariables::try_from(vars.clone())?) + } else { + Ok(CuddleVariables::try_from(vars.clone())?) + } + } + None => Err(anyhow::anyhow!( + "failed to find variables section in cuddle.yaml" + )), + } + } + .boxed() +} + +impl Component for CuddleVars { + fn name(&self) -> String { + "cuddle/vars".into() + } + + fn validate(&self, _value: &serde_yaml::Value) -> anyhow::Result<()> { + Ok(()) + } + + fn render_value(&self, _value: &serde_yaml::Value) -> Option> { + Some(Ok(minijinja::Value::from_object(self.variables.clone()))) + } +} + +impl minijinja::value::Object for CuddleVariables { + fn get_value(self: &std::sync::Arc, key: &minijinja::Value) -> Option { + if let Some(key) = key.as_str() { + tracing::trace!("looking up key: {}", key); + + if let Some(val) = self.0.get(key) { + match val { + CuddleVariable::Object(_) => todo!(), + CuddleVariable::Array(_) => todo!(), + CuddleVariable::String(str) => { + return Some(minijinja::Value::from_safe_string(str.to_owned())) + } + } + } + } + + None + } +} diff --git a/crates/cuddle-clusters/src/components.rs b/crates/cuddle-clusters/src/components.rs new file mode 100644 index 0000000..ac30479 --- /dev/null +++ b/crates/cuddle-clusters/src/components.rs @@ -0,0 +1,75 @@ +use std::rc::Rc; + +pub trait Component { + fn name(&self) -> String; + + fn validate(&self, _value: &serde_yaml::Value) -> anyhow::Result<()> { + Ok(()) + } + + fn render_value(&self, _value: &serde_yaml::Value) -> Option> { + None + } + + /// First return is name, second is contents + fn render(&self, _value: &serde_yaml::Value) -> Option> { + None + } +} + +#[derive(Clone)] +pub struct ConcreteComponent { + inner: Rc, +} + +impl std::ops::Deref for ConcreteComponent { + type Target = Rc; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl ConcreteComponent { + pub fn new(t: T) -> Self { + Self { inner: Rc::new(t) } + } +} + +pub trait IntoComponent { + fn into_component(self) -> ConcreteComponent; +} + +impl IntoComponent for ConcreteComponent { + fn into_component(self) -> ConcreteComponent { + self + } +} + +impl IntoComponent for T { + fn into_component(self) -> ConcreteComponent { + ConcreteComponent::new(self) + } +} + +#[cfg(test)] +mod test { + use similar_asserts::assert_eq; + + use super::*; + + pub struct Database {} + + impl Component for Database { + fn name(&self) -> String { + "cuddle/database".into() + } + } + + #[test] + fn can_transform_into_concrete() { + let database = Database {}; + + assert_eq!("cuddle/database", database.into_component().name()); + } +} diff --git a/crates/cuddle-clusters/src/lib.rs b/crates/cuddle-clusters/src/lib.rs index 9a818c6..8c6b2a6 100644 --- a/crates/cuddle-clusters/src/lib.rs +++ b/crates/cuddle-clusters/src/lib.rs @@ -1,3 +1,7 @@ -pub mod process; +pub mod components; +pub use components::*; +pub mod catalog; + +pub mod process; pub use process::{process, process_opts}; diff --git a/crates/cuddle-clusters/src/process.rs b/crates/cuddle-clusters/src/process.rs index 12f0dd7..f8001ec 100644 --- a/crates/cuddle-clusters/src/process.rs +++ b/crates/cuddle-clusters/src/process.rs @@ -9,8 +9,10 @@ use minijinja::context; use tokio::io::AsyncWriteExt; use tokio_stream::{wrappers::ReadDirStream, StreamExt}; +use crate::components::{Component, ConcreteComponent, IntoComponent}; + pub async fn process() -> anyhow::Result<()> { - process_opts(ProcessOpts::default()).await + process_opts(Vec::::new(), ProcessOpts::default()).await } pub struct ProcessOpts { @@ -30,7 +32,15 @@ impl Default for ProcessOpts { } } -pub async fn process_opts(opts: ProcessOpts) -> anyhow::Result<()> { +pub async fn process_opts( + components: impl IntoIterator, + opts: ProcessOpts, +) -> anyhow::Result<()> { + let components = components + .into_iter() + .map(|c| c.into_component()) + .collect::>(); + let path = opts.path.canonicalize().context("failed to find folder")?; let cuddle_path = path.join("cuddle.yaml"); @@ -51,7 +61,7 @@ pub async fn process_opts(opts: ProcessOpts) -> anyhow::Result<()> { tokio::fs::remove_dir_all(&opts.output).await?; tokio::fs::create_dir_all(&opts.output).await?; - process_templates(&clusters, &template_files, &opts.output).await?; + process_templates(&components, &clusters, &template_files, &opts.output).await?; Ok(()) } @@ -122,35 +132,47 @@ async fn read_dir(path: &Path) -> anyhow::Result> { } async fn process_templates( + components: &Vec, clusters: &CuddleClusters, template_files: &TemplateFiles, dest: &Path, ) -> anyhow::Result<()> { - for (cluster_name, _value) in clusters.iter() { - process_cluster(cluster_name, template_files, &dest.join(cluster_name)).await?; + for (environment, value) in clusters.iter() { + process_cluster( + &components, + value, + environment, + template_files, + &dest.join(environment), + ) + .await?; } Ok(()) } async fn process_cluster( - cluster_name: &str, + components: &Vec, + value: &serde_yaml::Value, + environment: &str, template_files: &TemplateFiles, dest: &Path, ) -> anyhow::Result<()> { for template_file in &template_files.templates { - process_template_file(cluster_name, template_file, dest).await?; + process_template_file(components, value, environment, template_file, dest).await?; } for raw_file in &template_files.raw { - process_raw_file(cluster_name, raw_file, dest).await?; + process_raw_file(environment, raw_file, dest).await?; } Ok(()) } async fn process_template_file( - _cluster_name: &str, + components: &Vec, + value: &serde_yaml::Value, + environment: &str, template_file: &PathBuf, dest: &Path, ) -> anyhow::Result<()> { @@ -175,8 +197,23 @@ async fn process_template_file( "failed to load template at: {}", template_file.display() ))?; + env.add_global("environment", environment); + + let mut variables = HashMap::new(); + for component in components { + let name = component.name(); + + if let Some(value) = component.render_value(value) { + let value = value?; + + variables.insert(name.replace("/", "_"), value); + } + } + let tmpl = env.get_template(file_name.to_str().unwrap_or_default())?; - let rendered = tmpl.render(context! {})?; + let rendered = tmpl.render(context! { + vars => variables + })?; dest_file.write_all(rendered.as_bytes()).await?; diff --git a/crates/cuddle-clusters/tests/common.rs b/crates/cuddle-clusters/tests/common.rs index 462a1ff..946a8c6 100644 --- a/crates/cuddle-clusters/tests/common.rs +++ b/crates/cuddle-clusters/tests/common.rs @@ -1,9 +1,12 @@ use std::{cmp::Ordering, path::Path}; -use cuddle_clusters::process::ProcessOpts; +use cuddle_clusters::{process::ProcessOpts, ConcreteComponent, IntoComponent}; use walkdir::DirEntry; -pub(crate) async fn run_test(name: &str) -> anyhow::Result<()> { +pub(crate) async fn run_test_with_components( + name: &str, + components: Vec, +) -> anyhow::Result<()> { let _ = tracing_subscriber::fmt::try_init(); println!("running for: {name}"); @@ -17,17 +20,25 @@ pub(crate) async fn run_test(name: &str) -> anyhow::Result<()> { let expected = test_folder.join("expected"); tokio::fs::create_dir_all(&expected).await?; - cuddle_clusters::process_opts(ProcessOpts { - path: test_folder.clone(), - output: actual.clone(), - }) + let components: Vec<_> = components.into_iter().map(|p| p.into_component()).collect(); + + cuddle_clusters::process_opts( + components.clone(), + ProcessOpts { + path: test_folder.clone(), + output: actual.clone(), + }, + ) .await?; if std::env::var("TEST_OVERRIDE") == Ok("true".to_string()) { - cuddle_clusters::process_opts(ProcessOpts { - path: test_folder, - output: expected.clone(), - }) + cuddle_clusters::process_opts( + components, + ProcessOpts { + path: test_folder, + output: expected.clone(), + }, + ) .await?; } @@ -36,6 +47,10 @@ pub(crate) async fn run_test(name: &str) -> anyhow::Result<()> { Ok(()) } +pub(crate) async fn run_test(name: &str) -> anyhow::Result<()> { + run_test_with_components(name, Vec::::new()).await +} + async fn compare(expected: &Path, actual: &Path) -> anyhow::Result<()> { let mut exp = walk_dir(expected)?; let mut act = walk_dir(actual)?; diff --git a/crates/cuddle-clusters/tests/cuddle_vars.rs b/crates/cuddle-clusters/tests/cuddle_vars.rs new file mode 100644 index 0000000..100c8e2 --- /dev/null +++ b/crates/cuddle-clusters/tests/cuddle_vars.rs @@ -0,0 +1,49 @@ +use std::collections::HashMap; + +use cuddle_clusters::catalog::cuddle_vars::{CuddleVariable, CuddleVariables, CuddleVars}; +use similar_asserts::assert_eq; + +#[tokio::test] +async fn can_load_cuddle_file() -> anyhow::Result<()> { + let current_dir = std::env::current_dir()?.join("tests"); + + let vars = CuddleVars::new(¤t_dir.join("cuddle_vars/basic/")).await?; + + assert_eq!( + CuddleVars { + variables: CuddleVariables(HashMap::from([ + ("service".into(), CuddleVariable::String("basic".into())), + ( + "some".into(), + CuddleVariable::Object(Box::new(CuddleVariables(HashMap::from([ + ("other".into(), CuddleVariable::String("item".into())), + ( + "nested".into(), + CuddleVariable::Object(Box::new(CuddleVariables(HashMap::from([( + "item".into(), + CuddleVariable::String("item".into()) + )])))) + ), + ( + "array".into(), + CuddleVariable::Array(vec![CuddleVariable::Object(Box::new( + CuddleVariables(HashMap::from([( + "item".into(), + CuddleVariable::Object(Box::new(CuddleVariables( + HashMap::from([( + "item".into(), + CuddleVariable::String("item".into()), + )]) + ))) + )])) + ))]) + ) + ])))) + ) + ])) + }, + vars + ); + + Ok(()) +} diff --git a/crates/cuddle-clusters/tests/cuddle_vars/basic/cuddle.yaml b/crates/cuddle-clusters/tests/cuddle_vars/basic/cuddle.yaml new file mode 100644 index 0000000..4bbf721 --- /dev/null +++ b/crates/cuddle-clusters/tests/cuddle_vars/basic/cuddle.yaml @@ -0,0 +1,10 @@ +vars: + service: basic + some: + other: item + nested: + item: item + array: + - item: + item: item + diff --git a/crates/cuddle-clusters/tests/tests.rs b/crates/cuddle-clusters/tests/tests.rs index 12bf9db..d7abbaa 100644 --- a/crates/cuddle-clusters/tests/tests.rs +++ b/crates/cuddle-clusters/tests/tests.rs @@ -1,8 +1,11 @@ pub mod common; mod can_run_for_env; +mod cuddle_vars; -use crate::common::run_test; +use cuddle_clusters::catalog::cuddle_vars::CuddleVars; + +use crate::common::{run_test, run_test_with_components}; #[tokio::test] async fn raw_files() -> anyhow::Result<()> { @@ -37,3 +40,16 @@ async fn environment() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test] +async fn with_cuddle_vars() -> anyhow::Result<()> { + let current_dir = std::env::current_dir()?.join("tests/with_cuddle_vars"); + + run_test_with_components( + "with_cuddle_vars", + vec![CuddleVars::new(¤t_dir).await?], + ) + .await?; + + Ok(()) +} diff --git a/crates/cuddle-clusters/tests/with_cuddle_vars/cuddle.yaml b/crates/cuddle-clusters/tests/with_cuddle_vars/cuddle.yaml new file mode 100644 index 0000000..7a13f5b --- /dev/null +++ b/crates/cuddle-clusters/tests/with_cuddle_vars/cuddle.yaml @@ -0,0 +1,5 @@ +vars: + service: service + +cuddle/clusters: + dev: diff --git a/crates/cuddle-clusters/tests/with_cuddle_vars/expected/dev/some.yaml b/crates/cuddle-clusters/tests/with_cuddle_vars/expected/dev/some.yaml new file mode 100644 index 0000000..e9a1947 --- /dev/null +++ b/crates/cuddle-clusters/tests/with_cuddle_vars/expected/dev/some.yaml @@ -0,0 +1,2 @@ +service: service + diff --git a/crates/cuddle-clusters/tests/with_cuddle_vars/templates/clusters/some.yaml.jinja2 b/crates/cuddle-clusters/tests/with_cuddle_vars/templates/clusters/some.yaml.jinja2 new file mode 100644 index 0000000..0f33df2 --- /dev/null +++ b/crates/cuddle-clusters/tests/with_cuddle_vars/templates/clusters/some.yaml.jinja2 @@ -0,0 +1,3 @@ +service: {{ vars.cuddle_vars.service }} + +