From 50d85ed383deccc0405a87737be8f2676edbed3d Mon Sep 17 00:00:00 2001 From: kjuulh Date: Thu, 23 May 2024 21:25:53 +0200 Subject: [PATCH] feat: add actual render Signed-off-by: kjuulh --- crates/cuddle-clusters/src/catalog.rs | 127 ++++++++++++++++++ crates/cuddle-clusters/src/process.rs | 64 +++++++-- crates/cuddle-clusters/tests/tests.rs | 19 ++- .../tests/with_vault_secrets/cuddle.yaml | 9 ++ .../with_vault_secrets/expected/dev/some.yaml | 39 ++++++ .../expected/dev/vault_secret.yaml | 13 ++ .../templates/clusters/some.yaml.jinja2 | 45 +++++++ 7 files changed, 302 insertions(+), 14 deletions(-) create mode 100644 crates/cuddle-clusters/tests/with_vault_secrets/cuddle.yaml create mode 100644 crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/some.yaml create mode 100644 crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/vault_secret.yaml create mode 100644 crates/cuddle-clusters/tests/with_vault_secrets/templates/clusters/some.yaml.jinja2 diff --git a/crates/cuddle-clusters/src/catalog.rs b/crates/cuddle-clusters/src/catalog.rs index 26cb49b..c6d7065 100644 --- a/crates/cuddle-clusters/src/catalog.rs +++ b/crates/cuddle-clusters/src/catalog.rs @@ -1,2 +1,129 @@ pub mod cluster_vars; pub mod cuddle_vars; +pub mod vault_secret { + use minijinja::{value::Object, Value}; + + use crate::Component; + + #[derive(Debug)] + pub struct VaultSecretValues { + name: String, + secrets: VaultSecretsLookup, + } + + #[derive(Debug, Clone)] + pub struct VaultSecretsLookup { + secrets: Vec, + } + + impl From> for VaultSecretsLookup { + fn from(value: Vec) -> Self { + Self { secrets: value } + } + } + + #[derive(Default, Clone)] + pub struct VaultSecret {} + + impl Component for VaultSecret { + fn name(&self) -> String { + "vault/secret".into() + } + + fn validate(&self, value: &serde_yaml::Value) -> anyhow::Result<()> { + Ok(()) + } + + fn render_value( + &self, + environment: &str, + value: &serde_yaml::Value, + ) -> Option> { + if let Some(values) = value + .as_mapping() + .and_then(|map| map.get("env")) + .and_then(|v| v.as_mapping()) + .map(|v| { + v.iter() + .filter_map(|(k, v)| { + if v.as_mapping() + .map(|m| m.get("vault").filter(|v| v.as_bool() == Some(true))) + .is_some() + { + Some(k) + } else { + None + } + }) + .filter_map(|k| k.as_str()) + .collect::>() + }) + { + let vault_values = VaultSecretValues { + name: self.name().replace("/", "-"), + secrets: values + .into_iter() + .map(|i| i.into()) + .collect::>() + .into(), + }; + + return Some(Ok(minijinja::Value::from_object(vault_values))); + } + + None + } + + fn render( + &self, + _environment: &str, + _value: &serde_yaml::Value, + ) -> Option> { + Some(Ok(( + format!("{}.yaml", self.name().replace("/", "_")), + r#"apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: {{ vars.cuddle_vars.service }}-{{ vars.vault_secret.name }} + namespace: {{ vars.cluster_vars.namespace }} +spec: + destination: + create: true + name: {{ vars.cuddle_vars.service }}-{{ vars.vault_secret.name }} + mount: kvv2 + path: {{ vars.cuddle_vars.service }}/{{ environment }} + refreshAfter: 30s + type: kv-v2 +"# + .into(), + ))) + } + } + + impl Object for VaultSecretValues { + fn get_value( + self: &std::sync::Arc, + key: &minijinja::Value, + ) -> Option { + let obj = match key.as_str()? { + "name" => Value::from_safe_string(self.name.clone()), + "secrets" => Value::from_object(self.secrets.clone()), + _ => return None, + }; + + Some(obj) + } + } + + impl Object for VaultSecretsLookup { + fn get_value(self: &std::sync::Arc, key: &Value) -> Option { + let idx = key.as_usize()?; + + self.secrets.get(idx).cloned().map(Value::from_safe_string) + } + + fn enumerate(self: &std::sync::Arc) -> minijinja::value::Enumerator { + minijinja::value::Enumerator::Seq(self.secrets.len()) + } + } +} diff --git a/crates/cuddle-clusters/src/process.rs b/crates/cuddle-clusters/src/process.rs index 83f162b..141cf90 100644 --- a/crates/cuddle-clusters/src/process.rs +++ b/crates/cuddle-clusters/src/process.rs @@ -10,7 +10,7 @@ use minijinja::context; use tokio::io::AsyncWriteExt; use tokio_stream::{wrappers::ReadDirStream, StreamExt}; -use crate::components::{Component, ConcreteComponent, IntoComponent}; +use crate::components::{ConcreteComponent, IntoComponent}; pub async fn process() -> anyhow::Result<()> { process_opts(Vec::::new(), ProcessOpts::default()).await @@ -210,6 +210,28 @@ async fn process_cluster( process_template_file(components, value, environment, template_file, dest).await?; } + for (template_file_name, template_content) in components + .iter() + .filter_map(|c| c.render(environment, value)) + .flat_map(|v| match v { + Ok(v) => Ok(v), + Err(e) => { + tracing::warn!("failed to render value for template"); + Err(e) + } + }) + { + process_render_template( + components, + value, + environment, + &template_file_name, + &template_content, + dest, + ) + .await?; + } + for (_, raw_file) in &template_files.raw { process_raw_file(environment, raw_file, dest).await?; } @@ -224,27 +246,43 @@ async fn process_template_file( template_file: &PathBuf, dest: &Path, ) -> anyhow::Result<()> { - // TODO: use mini jinja let file = tokio::fs::read_to_string(template_file) .await .context(format!("failed to find file: {}", template_file.display()))?; - - if !dest.exists() { - tokio::fs::create_dir_all(dest).await?; - } - let file_name = template_file .file_stem() .ok_or(anyhow::anyhow!("file didn't have a jinja2 format"))?; + process_render_template( + components, + value, + environment, + &file_name.to_string_lossy(), + &file, + dest, + ) + .await?; + + Ok(()) +} + +async fn process_render_template( + components: &Vec, + value: &serde_yaml::Value, + environment: &str, + file_name: &str, + file_content: &str, + dest: &Path, +) -> anyhow::Result<()> { + if !dest.exists() { + tokio::fs::create_dir_all(dest).await?; + } + let mut dest_file = tokio::fs::File::create(dest.join(file_name)).await?; let mut env = minijinja::Environment::new(); - env.add_template(file_name.to_str().unwrap_or_default(), &file) - .context(format!( - "failed to load template at: {}", - template_file.display() - ))?; + env.add_template(file_name, &file_content) + .context(format!("failed to load template at: {}", file_name))?; env.add_global("environment", environment); let mut variables = HashMap::new(); @@ -258,7 +296,7 @@ async fn process_template_file( } } - let tmpl = env.get_template(file_name.to_str().unwrap_or_default())?; + let tmpl = env.get_template(file_name)?; let rendered = tmpl.render(context! { vars => variables })?; diff --git a/crates/cuddle-clusters/tests/tests.rs b/crates/cuddle-clusters/tests/tests.rs index 12e9101..fc65335 100644 --- a/crates/cuddle-clusters/tests/tests.rs +++ b/crates/cuddle-clusters/tests/tests.rs @@ -4,7 +4,7 @@ mod can_run_for_env; mod cuddle_vars; use cuddle_clusters::{ - catalog::{cluster_vars::ClusterVars, cuddle_vars::CuddleVars}, + catalog::{cluster_vars::ClusterVars, cuddle_vars::CuddleVars, vault_secret::VaultSecret}, IntoComponent, }; @@ -79,3 +79,20 @@ async fn with_actual_deployment() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test] +async fn with_vault_secrets() -> anyhow::Result<()> { + let current_dir = std::env::current_dir()?.join("tests/with_vault_secrets"); + + run_test_with_components( + "with_vault_secrets", + vec![ + CuddleVars::new(¤t_dir).await?.into_component(), + ClusterVars::default().into_component(), + VaultSecret::default().into_component(), + ], + ) + .await?; + + Ok(()) +} diff --git a/crates/cuddle-clusters/tests/with_vault_secrets/cuddle.yaml b/crates/cuddle-clusters/tests/with_vault_secrets/cuddle.yaml new file mode 100644 index 0000000..5c9df8f --- /dev/null +++ b/crates/cuddle-clusters/tests/with_vault_secrets/cuddle.yaml @@ -0,0 +1,9 @@ +vars: + service: service + +cuddle/clusters: + dev: + env: + some.key: + vault: true + some.other.key: some_value diff --git a/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/some.yaml b/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/some.yaml new file mode 100644 index 0000000..e36f68e --- /dev/null +++ b/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/some.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: service + name: service +spec: + replicas: 3 + selector: + matchLabels: + app: service + template: + metadata: + labels: + app: service + spec: + containers: + - args: + - serve + command: + - service + image: kasperhermansen/service:main-1715336504 + name: service + envFrom: + - configMapRef: + name: service-config + env: + - name: SOME_KEY + valueFrom: + secretKeyRef: + name: service-vault-secret + key: some.key + ports: + - containerPort: 3000 + name: external-http + - containerPort: 3001 + name: internal-http + - containerPort: 3002 + name: internal-grpc \ No newline at end of file diff --git a/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/vault_secret.yaml b/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/vault_secret.yaml new file mode 100644 index 0000000..2e28f58 --- /dev/null +++ b/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/vault_secret.yaml @@ -0,0 +1,13 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: service-vault-secret + namespace: dev +spec: + destination: + create: true + name: service-vault-secret + mount: kvv2 + path: service/dev + refreshAfter: 30s + type: kv-v2 \ No newline at end of file diff --git a/crates/cuddle-clusters/tests/with_vault_secrets/templates/clusters/some.yaml.jinja2 b/crates/cuddle-clusters/tests/with_vault_secrets/templates/clusters/some.yaml.jinja2 new file mode 100644 index 0000000..af46d87 --- /dev/null +++ b/crates/cuddle-clusters/tests/with_vault_secrets/templates/clusters/some.yaml.jinja2 @@ -0,0 +1,45 @@ +{%- set service_name = vars.cuddle_vars.service -%} + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: {{ service_name }} + name: {{ service_name }} +spec: + replicas: 3 + selector: + matchLabels: + app: {{ service_name }} + template: + metadata: + labels: + app: {{ service_name }} + spec: + containers: + - args: + - serve + command: + - {{ service_name }} + image: kasperhermansen/{{ service_name }}:main-1715336504 + name: {{ service_name }} + envFrom: + - configMapRef: + name: {{service_name}}-config + {%- if vars.vault_secret is defined and (vars.vault_secret.secrets | length) > 0 %} + env: + {%- for secret in vars.vault_secret.secrets %} + - name: {{secret | upper | replace(".", "_") | replace("-", "_") }} + valueFrom: + secretKeyRef: + name: {{ service_name }}-{{ vars.vault_secret.name }} + key: {{ secret }} + {%- endfor %} + {%- endif %} + ports: + - containerPort: 3000 + name: external-http + - containerPort: 3001 + name: internal-http + - containerPort: 3002 + name: internal-grpc