From 3956366d28f41119d0fcef112ee89e26c3811555 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Fri, 29 Nov 2024 09:05:50 +0100 Subject: [PATCH] feat/add-postgres-database (#20) feat: add postgres database Signed-off-by: kjuulh feat: add postgres and more templates Signed-off-by: kjuulh Reviewed-on: https://git.front.kjuulh.io/kjuulh/cuddle-clusters/pulls/20 Co-authored-by: kjuulh Co-committed-by: kjuulh --- crates/cuddle-clusters/src/catalog.rs | 1 + .../src/catalog/cluster_vars.rs | 4 - .../src/catalog/postgres_database.rs | 146 ++++++++++++++++++ crates/cuddle-clusters/src/process.rs | 6 + .../can_run_for_env/expected/dev/test.yaml | 2 +- .../tests/environment/expected/dev/some.yaml | 2 +- .../tests/environment/expected/prod/some.yaml | 2 +- .../tests/jinja/expected/dev/some.yaml | 2 +- .../nested/expected/dev/service-template | 2 +- .../tests/nested/expected/dev/some-other.yaml | 2 +- .../tests/nested/expected/dev/some.yaml | 2 +- crates/cuddle-clusters/tests/tests.rs | 20 ++- .../with_crdb/expected/dev/cuddle-crdb.yaml | 2 +- .../tests/with_crdb/expected/dev/some.yaml | 2 +- .../with_ingress/expected/dev/ingress.yaml | 2 +- .../tests/with_ingress/expected/dev/some.yaml | 2 +- .../with_ingress/expected/prod/ingress.yaml | 2 +- .../with_ingress/expected/prod/some.yaml | 2 +- .../tests/with_postgres_database/cuddle.yaml | 7 + .../expected/dev/cuddle-postgres.yaml | 12 ++ .../expected/dev/some.yaml | 54 +++++++ .../templates/clusters/some.yaml.jinja2 | 56 +++++++ .../with_vault_secrets/expected/dev/some.yaml | 2 +- .../expected/dev/vault_secret.yaml | 2 +- 24 files changed, 316 insertions(+), 20 deletions(-) create mode 100644 crates/cuddle-clusters/src/catalog/postgres_database.rs create mode 100644 crates/cuddle-clusters/tests/with_postgres_database/cuddle.yaml create mode 100644 crates/cuddle-clusters/tests/with_postgres_database/expected/dev/cuddle-postgres.yaml create mode 100644 crates/cuddle-clusters/tests/with_postgres_database/expected/dev/some.yaml create mode 100644 crates/cuddle-clusters/tests/with_postgres_database/templates/clusters/some.yaml.jinja2 diff --git a/crates/cuddle-clusters/src/catalog.rs b/crates/cuddle-clusters/src/catalog.rs index d49afed..004c560 100644 --- a/crates/cuddle-clusters/src/catalog.rs +++ b/crates/cuddle-clusters/src/catalog.rs @@ -2,4 +2,5 @@ pub mod cluster_vars; pub mod crdb_database; pub mod cuddle_vars; pub mod ingress; +pub mod postgres_database; pub mod vault_secret; diff --git a/crates/cuddle-clusters/src/catalog/cluster_vars.rs b/crates/cuddle-clusters/src/catalog/cluster_vars.rs index 1ca9676..43eba2d 100644 --- a/crates/cuddle-clusters/src/catalog/cluster_vars.rs +++ b/crates/cuddle-clusters/src/catalog/cluster_vars.rs @@ -81,10 +81,6 @@ impl Component for ClusterVars { } } } - // vars.raw = match value.clone().try_into() { - // Ok(o) => o, - // Err(e) => panic!("{}", e), - // }; vars.raw = value.into(); vars.name = environment.into(); diff --git a/crates/cuddle-clusters/src/catalog/postgres_database.rs b/crates/cuddle-clusters/src/catalog/postgres_database.rs new file mode 100644 index 0000000..2cfc6c6 --- /dev/null +++ b/crates/cuddle-clusters/src/catalog/postgres_database.rs @@ -0,0 +1,146 @@ +use std::path::Path; + +use minijinja::{value::Object, Value}; + +use crate::Component; + +use super::cuddle_vars::{load_cuddle_file, CuddleVariable, CuddleVariables}; + +pub struct PostgresDatabase { + variables: CuddleVariables, +} + +impl PostgresDatabase { + pub async fn new(path: &Path) -> anyhow::Result { + let variables = load_cuddle_file(path).await?; + + Ok(Self { variables }) + } +} + +impl Component for PostgresDatabase { + fn name(&self) -> String { + "cuddle/postgres".into() + } + + fn render_value( + &self, + environment: &str, + _value: &serde_yaml::Value, + ) -> Option> { + if let Some(true) = self + .variables + .0 + .get("database") + .and_then(|v| match v { + CuddleVariable::Object(o) => Some(o), + _ => None, + }) + .and_then(|o| o.0.get("postgres")) + .and_then(|o| match o { + CuddleVariable::String(o) => { + if o == "true" { + Some(true) + } else { + None + } + } + _ => None, + }) + { + return Some(Ok(minijinja::Value::from_object(PostgresDatabaseValues { + name: self.name(), + enabled: true, + }))); + } + + Some(Ok(minijinja::Value::from_object(PostgresDatabaseValues { + name: self.name(), + enabled: false, + }))) + } + + fn render( + &self, + environment: &str, + _value: &serde_yaml::Value, + ) -> Option> { + if let Some(true) = self + .variables + .0 + .get("database") + .and_then(|v| match v { + CuddleVariable::Object(o) => Some(o), + _ => None, + }) + .and_then(|o| o.0.get("postgres")) + .and_then(|o| match o { + CuddleVariable::String(o) => { + if o == "true" { + Some(true) + } else { + None + } + } + _ => None, + }) + { + return Some(Ok(( + format!("{}.yaml", self.name().replace("/", "-")), + r#" +{%- if environment == "dev" %} +{%- set port = 5433 %} +{%- else %} +{%- set port = 5432 %} +{%- endif %} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ vars.cuddle_postgres.file_name(vars.cuddle_vars.service) }} + namespace: {{ vars.cluster_vars.namespace }} +data: + DATABASE_TYPE: postgresql + DATABASE_HOST: {{ environment }}.postgresql.kjuulh.app + DATABASE_PORT: {{ port }} + DATABASE_USER: {{ vars.cuddle_vars.service | replace("_", "-") }} + DATABASE_DB: {{ vars.cuddle_vars.service | replace("_", "-") }} + +"# + .into(), + ))); + } + + None + } +} + +#[derive(Debug)] +struct PostgresDatabaseValues { + name: String, + enabled: bool, +} + +impl Object for PostgresDatabaseValues { + fn get_value(self: &std::sync::Arc, key: &minijinja::Value) -> Option { + let name = self.name.clone(); + match key.as_str()? { + "has_values" => { + if self.enabled { + Some(minijinja::Value::from_serialize(true)) + } else { + Some(minijinja::Value::from_serialize(false)) + } + } + "file_name" => Some(Value::from_function(move |file_name: String| { + format!("{}-{}", file_name, name.replace("/", "-")) + })), + "env" => Some(Value::from_serialize(vec![ + "DATABASE_HOST", + "DATABASE_PORT", + "DATABASE_USER", + "DATABASE_DB", + ])), + _ => None, + } + } +} diff --git a/crates/cuddle-clusters/src/process.rs b/crates/cuddle-clusters/src/process.rs index 7348e1a..976d300 100644 --- a/crates/cuddle-clusters/src/process.rs +++ b/crates/cuddle-clusters/src/process.rs @@ -354,6 +354,12 @@ async fn process_render_template( vars => variables })?; + let rendered = if rendered.is_empty() || rendered.ends_with("\n") { + rendered + } else { + format!("{rendered}\n") + }; + dest_file.write_all(rendered.as_bytes()).await?; Ok(()) diff --git a/crates/cuddle-clusters/tests/can_run_for_env/expected/dev/test.yaml b/crates/cuddle-clusters/tests/can_run_for_env/expected/dev/test.yaml index 92dbfb6..5418072 100644 --- a/crates/cuddle-clusters/tests/can_run_for_env/expected/dev/test.yaml +++ b/crates/cuddle-clusters/tests/can_run_for_env/expected/dev/test.yaml @@ -3,4 +3,4 @@ some = { thing = "some" } -} \ No newline at end of file +} diff --git a/crates/cuddle-clusters/tests/environment/expected/dev/some.yaml b/crates/cuddle-clusters/tests/environment/expected/dev/some.yaml index 4a067a0..b8500d2 100644 --- a/crates/cuddle-clusters/tests/environment/expected/dev/some.yaml +++ b/crates/cuddle-clusters/tests/environment/expected/dev/some.yaml @@ -1 +1 @@ -env: dev \ No newline at end of file +env: dev diff --git a/crates/cuddle-clusters/tests/environment/expected/prod/some.yaml b/crates/cuddle-clusters/tests/environment/expected/prod/some.yaml index 46b998b..2fdef9a 100644 --- a/crates/cuddle-clusters/tests/environment/expected/prod/some.yaml +++ b/crates/cuddle-clusters/tests/environment/expected/prod/some.yaml @@ -1 +1 @@ -env: prod \ No newline at end of file +env: prod diff --git a/crates/cuddle-clusters/tests/jinja/expected/dev/some.yaml b/crates/cuddle-clusters/tests/jinja/expected/dev/some.yaml index f8c3979..b9c3a29 100644 --- a/crates/cuddle-clusters/tests/jinja/expected/dev/some.yaml +++ b/crates/cuddle-clusters/tests/jinja/expected/dev/some.yaml @@ -1 +1 @@ -some_file: 4 \ No newline at end of file +some_file: 4 diff --git a/crates/cuddle-clusters/tests/nested/expected/dev/service-template b/crates/cuddle-clusters/tests/nested/expected/dev/service-template index ce93fac..24e1098 100644 --- a/crates/cuddle-clusters/tests/nested/expected/dev/service-template +++ b/crates/cuddle-clusters/tests/nested/expected/dev/service-template @@ -1 +1 @@ -service \ No newline at end of file +service diff --git a/crates/cuddle-clusters/tests/nested/expected/dev/some-other.yaml b/crates/cuddle-clusters/tests/nested/expected/dev/some-other.yaml index ce93fac..24e1098 100644 --- a/crates/cuddle-clusters/tests/nested/expected/dev/some-other.yaml +++ b/crates/cuddle-clusters/tests/nested/expected/dev/some-other.yaml @@ -1 +1 @@ -service \ No newline at end of file +service diff --git a/crates/cuddle-clusters/tests/nested/expected/dev/some.yaml b/crates/cuddle-clusters/tests/nested/expected/dev/some.yaml index ce93fac..24e1098 100644 --- a/crates/cuddle-clusters/tests/nested/expected/dev/some.yaml +++ b/crates/cuddle-clusters/tests/nested/expected/dev/some.yaml @@ -1 +1 @@ -service \ No newline at end of file +service diff --git a/crates/cuddle-clusters/tests/tests.rs b/crates/cuddle-clusters/tests/tests.rs index 5f4208f..4a34df1 100644 --- a/crates/cuddle-clusters/tests/tests.rs +++ b/crates/cuddle-clusters/tests/tests.rs @@ -6,7 +6,7 @@ mod cuddle_vars; use cuddle_clusters::{ catalog::{ cluster_vars::ClusterVars, crdb_database::CockroachDB, cuddle_vars::CuddleVars, - ingress::Ingress, vault_secret::VaultSecret, + ingress::Ingress, postgres_database::PostgresDatabase, vault_secret::VaultSecret, }, IntoComponent, }; @@ -125,3 +125,21 @@ async fn with_ingress() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test] +async fn with_postgres_databse() -> anyhow::Result<()> { + let current_dir = std::env::current_dir()?.join("tests/with_postgres_database"); + + run_test_with_components( + "with_postgres_database", + vec![ + CuddleVars::new(¤t_dir).await?.into_component(), + ClusterVars::default().into_component(), + VaultSecret::default().into_component(), + PostgresDatabase::new(¤t_dir).await?.into_component(), + ], + ) + .await?; + + Ok(()) +} diff --git a/crates/cuddle-clusters/tests/with_crdb/expected/dev/cuddle-crdb.yaml b/crates/cuddle-clusters/tests/with_crdb/expected/dev/cuddle-crdb.yaml index 8a380ff..35a1cac 100644 --- a/crates/cuddle-clusters/tests/with_crdb/expected/dev/cuddle-crdb.yaml +++ b/crates/cuddle-clusters/tests/with_crdb/expected/dev/cuddle-crdb.yaml @@ -6,4 +6,4 @@ metadata: namespace: dev data: DATABASE_URL: postgresql://root@dev-cluster:26257/service - \ No newline at end of file + diff --git a/crates/cuddle-clusters/tests/with_crdb/expected/dev/some.yaml b/crates/cuddle-clusters/tests/with_crdb/expected/dev/some.yaml index 6286958..0db2e5a 100644 --- a/crates/cuddle-clusters/tests/with_crdb/expected/dev/some.yaml +++ b/crates/cuddle-clusters/tests/with_crdb/expected/dev/some.yaml @@ -36,4 +36,4 @@ spec: - containerPort: 3001 name: internal-http - containerPort: 3002 - name: internal-grpc \ No newline at end of file + name: internal-grpc diff --git a/crates/cuddle-clusters/tests/with_ingress/expected/dev/ingress.yaml b/crates/cuddle-clusters/tests/with_ingress/expected/dev/ingress.yaml index b7d7d60..5fe644c 100644 --- a/crates/cuddle-clusters/tests/with_ingress/expected/dev/ingress.yaml +++ b/crates/cuddle-clusters/tests/with_ingress/expected/dev/ingress.yaml @@ -118,4 +118,4 @@ spec: tls: - hosts: - grpc.service.dev.internal.kjuulh.app - secretName: tls-service-kjuulh-app-internal-grpc-ingress-dns \ No newline at end of file + secretName: tls-service-kjuulh-app-internal-grpc-ingress-dns diff --git a/crates/cuddle-clusters/tests/with_ingress/expected/dev/some.yaml b/crates/cuddle-clusters/tests/with_ingress/expected/dev/some.yaml index 43c5173..87146b9 100644 --- a/crates/cuddle-clusters/tests/with_ingress/expected/dev/some.yaml +++ b/crates/cuddle-clusters/tests/with_ingress/expected/dev/some.yaml @@ -27,4 +27,4 @@ spec: - containerPort: 3001 name: internal-http - containerPort: 3002 - name: internal-grpc \ No newline at end of file + name: internal-grpc diff --git a/crates/cuddle-clusters/tests/with_ingress/expected/prod/ingress.yaml b/crates/cuddle-clusters/tests/with_ingress/expected/prod/ingress.yaml index 0847f5f..b738d43 100644 --- a/crates/cuddle-clusters/tests/with_ingress/expected/prod/ingress.yaml +++ b/crates/cuddle-clusters/tests/with_ingress/expected/prod/ingress.yaml @@ -118,4 +118,4 @@ spec: tls: - hosts: - grpc.service.prod.internal.kjuulh.app - secretName: tls-service-kjuulh-app-internal-grpc-ingress-dns \ No newline at end of file + secretName: tls-service-kjuulh-app-internal-grpc-ingress-dns diff --git a/crates/cuddle-clusters/tests/with_ingress/expected/prod/some.yaml b/crates/cuddle-clusters/tests/with_ingress/expected/prod/some.yaml index 43c5173..87146b9 100644 --- a/crates/cuddle-clusters/tests/with_ingress/expected/prod/some.yaml +++ b/crates/cuddle-clusters/tests/with_ingress/expected/prod/some.yaml @@ -27,4 +27,4 @@ spec: - containerPort: 3001 name: internal-http - containerPort: 3002 - name: internal-grpc \ No newline at end of file + name: internal-grpc diff --git a/crates/cuddle-clusters/tests/with_postgres_database/cuddle.yaml b/crates/cuddle-clusters/tests/with_postgres_database/cuddle.yaml new file mode 100644 index 0000000..40ddb91 --- /dev/null +++ b/crates/cuddle-clusters/tests/with_postgres_database/cuddle.yaml @@ -0,0 +1,7 @@ +vars: + service: service + database: + postgres: "true" + +cuddle/clusters: + dev: diff --git a/crates/cuddle-clusters/tests/with_postgres_database/expected/dev/cuddle-postgres.yaml b/crates/cuddle-clusters/tests/with_postgres_database/expected/dev/cuddle-postgres.yaml new file mode 100644 index 0000000..dbef0ec --- /dev/null +++ b/crates/cuddle-clusters/tests/with_postgres_database/expected/dev/cuddle-postgres.yaml @@ -0,0 +1,12 @@ + +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-cuddle-postgres + namespace: dev +data: + DATABASE_TYPE: postgresql + DATABASE_HOST: dev.postgresql.kjuulh.app + DATABASE_PORT: 5433 + DATABASE_USER: service + DATABASE_DB: service diff --git a/crates/cuddle-clusters/tests/with_postgres_database/expected/dev/some.yaml b/crates/cuddle-clusters/tests/with_postgres_database/expected/dev/some.yaml new file mode 100644 index 0000000..ba5d876 --- /dev/null +++ b/crates/cuddle-clusters/tests/with_postgres_database/expected/dev/some.yaml @@ -0,0 +1,54 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: service + name: service +spec: + replicas: 3 + selector: + matchLabels: + app: service + template: + metadata: + labels: + app: service + spec: + containers: + - args: + - serve + command: + - service + image: kasperhermansen/service:main-1715336504 + name: service + envFrom: + - configMapRef: + name: service-config + env: + - name: DATABASE_HOST + valueFrom: + secretKeyRef: + name: service-cuddle-postgres + key: DATABASE_HOST + - name: DATABASE_PORT + valueFrom: + secretKeyRef: + name: service-cuddle-postgres + key: DATABASE_PORT + - name: DATABASE_USER + valueFrom: + secretKeyRef: + name: service-cuddle-postgres + key: DATABASE_USER + - name: DATABASE_DB + valueFrom: + secretKeyRef: + name: service-cuddle-postgres + key: DATABASE_DB + ports: + - containerPort: 3000 + name: external-http + - containerPort: 3001 + name: internal-http + - containerPort: 3002 + name: internal-grpc diff --git a/crates/cuddle-clusters/tests/with_postgres_database/templates/clusters/some.yaml.jinja2 b/crates/cuddle-clusters/tests/with_postgres_database/templates/clusters/some.yaml.jinja2 new file mode 100644 index 0000000..c9dff3c --- /dev/null +++ b/crates/cuddle-clusters/tests/with_postgres_database/templates/clusters/some.yaml.jinja2 @@ -0,0 +1,56 @@ +{%- set service_name = vars.cuddle_vars.service -%} + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: {{ service_name }} + name: {{ service_name }} +spec: + replicas: 3 + selector: + matchLabels: + app: {{ service_name }} + template: + metadata: + labels: + app: {{ service_name }} + spec: + containers: + - args: + - serve + command: + - {{ service_name }} + image: kasperhermansen/{{ service_name }}:main-1715336504 + name: {{ service_name }} + envFrom: + - configMapRef: + name: {{service_name}}-config + {%- if vars.vault_secret.has_values or vars.cuddle_postgres.has_values %} + env: + {%- if vars.vault_secret.has_values %} + {%- for secret in vars.vault_secret.secrets %} + - name: {{secret | upper | replace(".", "_") | replace("-", "_") }} + valueFrom: + secretKeyRef: + name: {{ vars.vault_secret.file_name(service_name) }} + key: {{ secret }} + {%- endfor %} + {%- endif %} + {%- if vars.cuddle_postgres.has_values %} + {%- for env in vars.cuddle_postgres.env %} + - name: {{ env }} + valueFrom: + secretKeyRef: + name: {{ vars.cuddle_postgres.file_name(service_name) }} + key: {{ env }} + {%- endfor %} + {%- endif %} + {%- endif %} + ports: + - containerPort: 3000 + name: external-http + - containerPort: 3001 + name: internal-http + - containerPort: 3002 + name: internal-grpc diff --git a/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/some.yaml b/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/some.yaml index e36f68e..3aabbe6 100644 --- a/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/some.yaml +++ b/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/some.yaml @@ -36,4 +36,4 @@ spec: - containerPort: 3001 name: internal-http - containerPort: 3002 - name: internal-grpc \ No newline at end of file + name: internal-grpc diff --git a/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/vault_secret.yaml b/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/vault_secret.yaml index 2e28f58..d8c93eb 100644 --- a/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/vault_secret.yaml +++ b/crates/cuddle-clusters/tests/with_vault_secrets/expected/dev/vault_secret.yaml @@ -10,4 +10,4 @@ spec: mount: kvv2 path: service/dev refreshAfter: 30s - type: kv-v2 \ No newline at end of file + type: kv-v2