Compare commits

..

1 Commits

Author SHA1 Message Date
991825df95
feat: add resources
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
Signed-off-by: kjuulh <contact@kjuulh.io>
2024-03-09 23:02:43 +01:00
19 changed files with 451 additions and 3840 deletions

View File

@ -47,6 +47,8 @@ steps:
volumes:
- name: ssh
path: /root/.ssh/
- name: dockersock
path: /var/run
- name: ci
path: /mnt/ci
commands:
@ -54,21 +56,13 @@ steps:
- ssh-add
- echo "$DOCKER_PASSWORD" | docker login --password-stdin --username="$DOCKER_USERNAME" docker.io
- apk add git
- cuddle --version
- $CI_PREFIX pr
environment:
DAGGER_CLOUD_TOKEN:
from_secret: dagger_cloud_token
DRONE_HOST: "https://ci.i.kjuulh.io"
DRONE_USER: "kjuulh"
DRONE_TOKEN:
from_secret: drone_token
DOCKER_BUILDKIT: 1
DOCKER_PASSWORD:
from_secret: docker_password
DOCKER_USERNAME:
from_secret: docker_username
DOCKER_HOST: "tcp://192.168.1.155:2376"
CUDDLE_SECRETS_PROVIDER: 1password
CUDDLE_ONE_PASSWORD_DOT_ENV: ".env.ci"
CUDDLE_SSH_AGENT: "true"
@ -93,6 +87,8 @@ steps:
volumes:
- name: ssh
path: /root/.ssh/
# - name: dockersock
# path: /var/run
- name: ci
path: /mnt/ci
commands:
@ -100,15 +96,8 @@ steps:
- ssh-add
- echo "$DOCKER_PASSWORD" | docker login --password-stdin --username="$DOCKER_USERNAME" docker.io
- apk add git
- cuddle --version
- $CI_PREFIX main
environment:
DAGGER_CLOUD_TOKEN:
from_secret: dagger_cloud_token
DRONE_HOST: "https://ci.i.kjuulh.io"
DRONE_USER: "kjuulh"
DRONE_TOKEN:
from_secret: drone_token
REGISTRY_CACHE_USERNAME:
from_secret: registry_cache_username
REGISTRY_CACHE_PASSWORD:
@ -128,7 +117,7 @@ steps:
GIT_PASSWORD:
from_secret: git_password
CI_PREFIX: "/mnt/ci/ci"
DOCKER_HOST: "tcp://192.168.1.155:2376"
DOCKER_HOST: "tcp://192.168.1.233:2376"
CUDDLE_PLEASE_TOKEN:
from_secret: cuddle_please_token
OP_SERVICE_ACCOUNT_TOKEN:
@ -151,6 +140,8 @@ steps:
volumes:
- name: ssh
path: /root/.ssh/
- name: dockersock
path: /var/run
- name: ci
path: /mnt/ci
commands:
@ -184,8 +175,18 @@ steps:
- "load_secret"
- "build ci"
services:
- name: docker
image: docker:dind
privileged: true
volumes:
- name: dockersock
path: /var/run
volumes:
- name: ssh
temp: {}
- name: dockersock
temp: {}
- name: ci
temp: {}

3640
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -4,11 +4,9 @@ resolver = "2"
[workspace.dependencies]
anyhow = { version = "1.0.86" }
anyhow = { version = "1.0.80" }
tokio = { version = "1", features = ["full"] }
tracing = { version = "0.1", features = ["log"] }
tracing-subscriber = { version = "0.3.18" }
clap = { version = "4.5.4", features = ["derive", "env"] }
clap = { version = "4.5.2", features = ["derive", "env"] }
dotenv = { version = "0.15.0" }
cuddle-clusters = { git = "https://git.front.kjuulh.io/kjuulh/cuddle-clusters", branch = "main" } #tag = "v0.1.1" }

View File

@ -8,7 +8,7 @@ edition = "2021"
[dependencies]
tokio.workspace = true
dagger-sdk = "0.13.7"
dagger-sdk = {git = "https://github.com/kjuulh/dagger.git", branch = "feat/with-send-sync"}
eyre = { version = "0.6.12" }
dagger-components = { git = "https://git.front.kjuulh.io/kjuulh/dagger-components", branch = "main" }

View File

@ -1,17 +1,15 @@
use std::sync::Arc;
use cuddle_ci::drone_templater::DroneTemplater;
use cuddle_ci::rust_service::architecture::{Architecture, Os};
use cuddle_ci::rust_service::{extensions::*, RustService};
use cuddle_ci::CuddleCI;
use tokio::sync::Mutex;
const BIN_NAME: &str = "cuddle-rust-service-plan";
#[tokio::main]
async fn main() -> eyre::Result<()> {
dagger_sdk::connect(|client| async move {
let service = &RustService::from(client.clone())
let client = dagger_sdk::connect().await?;
let service = RustService::from(client)
.with_arch(Architecture::Amd64)
.with_os(Os::Linux)
.with_apt(&[
@ -21,33 +19,25 @@ async fn main() -> eyre::Result<()> {
"libgit2-dev",
"git",
"openssh-client",
"protobuf-compiler",
])
.with_apt_release(&["git", "openssh-client", "protobuf-compiler"])
.with_apt_release(&["git", "openssh-client"])
.with_docker_cli()
.with_cuddle_cli()
.with_kubectl()
.with_apt_ca_certificates()
.with_crates(["ci", "crates/*"])
.with_mold("2.3.3")
.with_bin_name(BIN_NAME)
.with_bin_name("cuddle-rust-service-plan")
.with_deployment(false)
.to_owned();
let drone_templater =
&DroneTemplater::new(client, "templates/cuddle-rust-service-plan.yaml")
.with_variable("bin_name", BIN_NAME)
.to_owned();
let service = Arc::new(Mutex::new(service));
CuddleCI::default()
.with_pull_request(service)
.with_main(service)
.with_main(drone_templater)
.with_pull_request(service.clone())
.with_main(service.clone())
.execute(std::env::args())
.await?;
Ok(())
})
.await?;
Ok(())
}

View File

@ -6,14 +6,10 @@ edition = "2021"
[dependencies]
tokio.workspace = true
dagger-sdk = "0.13.7"
dagger-sdk = {git = "https://github.com/kjuulh/dagger.git", branch = "feat/with-send-sync"}
eyre = { version = "0.6.12" }
tracing.workspace = true
tracing-subscriber.workspace = true
dagger-components = { git = "https://git.front.kjuulh.io/kjuulh/dagger-components", branch = "main" }
dagger-rust = { git = "https://git.front.kjuulh.io/kjuulh/dagger-components", branch = "main" }
cuddle-ci = { git = "https://git.front.kjuulh.io/kjuulh/dagger-components", branch = "main" }
cuddle-clusters.workspace = true
async-trait = "0.1.80"
async-trait = "0.1.77"

View File

@ -1,183 +1,51 @@
use std::collections::HashMap;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use std::sync::Arc;
use async_trait::async_trait;
use cuddle_ci::cuddle_file::CuddleFile;
use cuddle_ci::cuddle_please::CuddlePlease;
use cuddle_ci::cuddle_releaser::CuddleReleaser;
use cuddle_ci::rust_service::architecture::{Architecture, Os};
use cuddle_ci::rust_service::RustService;
use cuddle_ci::rust_service::{extensions::*, RustServiceContext};
use cuddle_ci::{Context, CuddleCI, MainAction, PullRequestAction};
use cuddle_clusters::catalog::cluster_vars::ClusterVars;
use cuddle_clusters::catalog::crdb_database::CockroachDB;
use cuddle_clusters::catalog::cuddle_vars::CuddleVars;
use cuddle_clusters::catalog::ingress::Ingress;
use cuddle_clusters::catalog::vault_secret::VaultSecret;
use cuddle_clusters::releaser::Releaser;
use cuddle_clusters::IntoComponent;
use tokio::sync::Mutex;
#[tokio::main]
async fn main() -> eyre::Result<()> {
tracing_subscriber::fmt::init();
dagger_sdk::connect(|client| async move {
let client = dagger_sdk::connect().await?;
let cuddle_file = CuddleFile::from_cuddle_file().await?;
let service = &RustService::from(client.clone())
.with_cuddle_file(&cuddle_file)
let service = RustService::from(client.clone())
.with_arch(Architecture::Amd64)
.with_os(Os::Linux)
.with_apt(&["libssl-dev", "libz-dev", "libpq-dev", "protobuf-compiler"])
.with_apt_release(&["libssl-dev", "libz-dev", "libpq-dev"])
.with_apt_ca_certificates()
.with_workspace_crates()
.await
.with_crates(["crates/*"])
.with_mold("2.3.3")
.with_bin_name(cuddle_file.vars.service)
.with_deployment(false)
.to_owned();
let render = &RustServiceRender {
service: cuddle_file.vars.service.clone(),
//registry: "http://127.0.0.1:7900".into(),
//registry: "http://10.0.11.19:7900".into(),
registry: "https://releaser.i.kjuulh.io:443".into(),
};
let deployment = &CuddleReleaser::new(client.clone()).await?;
let service = Arc::new(Mutex::new(service));
let mut ci = CuddleCI::default();
let render = Arc::new(Mutex::new(RustServiceRender::default()));
ci.with_pull_request(service)
.with_pull_request(render)
let deployment = Arc::new(Mutex::new(CuddleReleaser::new(client).await?));
CuddleCI::default()
.with_pull_request(service.clone())
//.with_pull_request(render.clone())
//.with_pull_request(deployment.clone())
.with_main(service)
.with_main(render)
.with_main(deployment);
if cuddle_file.please.is_some() {
ci.with_main(&CuddlePlease::new(client.clone()));
}
let name = cuddle_file.vars.service.clone();
let system_time = SystemTime::now().duration_since(UNIX_EPOCH)?;
let build_notifications = client
.container()
.from("docker.io/kasperhermansen/build-notifications:main-1731768496")
.with_env_variable("TIME", system_time.as_secs().to_string())
.with_env_variable(
"SERVICE_HOST",
"https://build-notifications.prod.internal.kjuulh.app",
)
.with_env_variable(
"SERVICE_GRPC_HOST",
"https://grpc.build-notifications.prod.internal.kjuulh.app",
);
if let Err(e) = build_notifications
.with_exec(vec![
"build-notifications",
"build-started",
"--project-name",
&name,
])
.sync()
.await
{
tracing::warn!("failed to send start notification: {}", e.to_string())
}
match ci.execute(std::env::args()).await {
Ok(()) => {
if let Err(e) = build_notifications
.with_exec(vec![
"build-notifications",
"build-success",
"--project-name",
&name,
])
.sync()
.await
{
tracing::warn!("failed to send success notification: {}", e.to_string())
}
}
Err(e) => {
if let Err(e) = build_notifications
.with_exec(vec![
"build-notifications",
"build-failure",
"--project-name",
&name,
"--error",
&e.to_string(),
])
.sync()
.await
{
tracing::warn!("failed to send failure notification: {}", e.to_string())
}
return Err(e);
}
}
Ok(())
})
.with_main(service.clone())
.with_main(render.clone())
.with_main(deployment.clone())
.execute(std::env::args())
.await?;
Ok(())
}
#[derive(Default, Clone)]
struct RustServiceRender {
service: String,
registry: String,
}
impl RustServiceRender {
async fn render_templates(&self, image_tag: &str) -> eyre::Result<()> {
let mut releaser = Releaser::default();
releaser
.with_service(&self.service)
.with_registry(&self.registry);
match cuddle_clusters::process_opts(
vec![
CuddleVars::new(&std::env::current_dir()?)
.await
.map_err(|e| eyre::anyhow!("failed to get cuddle vars: {}", e.to_string()))?
.into_component(),
ClusterVars::default().into_component(),
VaultSecret::default().into_component(),
CockroachDB::new(&std::env::current_dir()?)
.await
.map_err(|e| eyre::anyhow!("failed to get cuddle vars: {}", e.to_string()))?
.into_component(),
Ingress::new(&std::env::current_dir()?)
.await
.map_err(|e| eyre::anyhow!("failed to get cuddle vars: {}", e.to_string()))?
.into_component(),
],
cuddle_clusters::process::ProcessOpts {
path: std::env::current_dir()?,
output: PathBuf::from(".cuddle/tmp/cuddle-clusters"),
variables: HashMap::from([("image_tag".into(), image_tag.into())]),
},
Some(releaser),
)
.await
.map_err(|e| eyre::anyhow!("failed to process templates: {}", e.to_string()))
{
Ok(_) => {}
Err(e) => {
tracing::error!("failed to process templates: {}", e);
}
}
Ok(())
}
}
#[derive(Default)]
struct RustServiceRender {}
#[async_trait]
impl MainAction for RustServiceRender {
@ -186,15 +54,13 @@ impl MainAction for RustServiceRender {
.get_image_tag()?
.ok_or(eyre::anyhow!("failed to find image_tag"))?;
self.render_templates(&image_tag).await?;
// cuddle_ci::cuddle_x::well_known::render(vec![
// "--cluster",
// "clank-prod",
// "--image_tag",
// &image_tag,
// ])
// .await?;
cuddle_ci::cuddle_x::well_known::render(vec![
"--cluster",
"clank-prod",
"--image_tag",
&image_tag,
])
.await?;
Ok(())
}
@ -207,7 +73,13 @@ impl PullRequestAction for RustServiceRender {
.get_image_tag()?
.ok_or(eyre::anyhow!("failed to find image_tag"))?;
self.render_templates(&image_tag).await?;
cuddle_ci::cuddle_x::well_known::render(vec![
"--cluster",
"clank-dev",
"--image_tag",
&image_tag,
])
.await?;
Ok(())
}

View File

@ -10,13 +10,6 @@ vars:
clank-prod:
replicas: "3"
namespace: prod
config:
something: something
secrets:
something.else: something
cuddle/clusters:
dev:
scripts:
render:

View File

@ -1,13 +0,0 @@
{%- set cluster_namespace = vars.cluster_vars.namespace -%}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ vars.cuddle_vars.service }}-config
namespace: {{ cluster_namespace }}
data:
{%- if (vars.cluster_vars.env | items | length) > 0 %}
{%- for (name, value) in vars.cluster_vars.env | dictsort %}
{{name | upper | replace(".", "_") | replace("-", "_") }}: {{value}}
{%- endfor %}
{%- endif %}

View File

@ -1,68 +0,0 @@
{%- set service_name = vars.cuddle_vars.service -%}
{%- set cluster_name = vars.cluster_vars.name -%}
{%- set cluster_namespace = vars.cluster_vars.namespace -%}
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: {{ service_name }}
cluster: {{ cluster_name }}
name: {{ service_name }}
namespace: {{ cluster_namespace }}
spec:
replicas: {{ vars.cluster_vars.replicas }}
selector:
matchLabels:
app: {{ service_name }}
cluster: {{ cluster_name }}
template:
metadata:
labels:
app: {{ service_name }}
cluster: {{ cluster_name }}
spec:
containers:
- args:
- serve
command:
- {{ service_name }}
image: kasperhermansen/{{ service_name }}:{{ vars.user_vars.image_tag }}
name: {{ service_name }}
envFrom:
- configMapRef:
name: {{service_name}}-config
{%- if vars.cuddle_crdb.has_values %}
- configMapRef:
name: {{ vars.cuddle_crdb.file_name(service_name) }}
{%- endif %}
{%- if vars.vault_secret.has_values or vars.cuddle_crdb.has_values %}
env:
{%- if vars.vault_secret.has_values %}
{%- for secret in vars.vault_secret.secrets %}
- name: {{secret | upper | replace(".", "_") | replace("-", "_") }}
valueFrom:
secretKeyRef:
name: {{ vars.vault_secret.file_name(service_name) }}
key: {{ secret }}
{%- endfor %}
{%- endif %}
{#
{%- if vars.cuddle_crdb.has_values %}
- name: {{vars.cuddle_crdb.env }}
valueFrom:
secretKeyRef:
name: {{ vars.cuddle_crdb.file_name(service_name) }}
key: {{ vars.cuddle_crdb.env }}
{%- endif %}
#}
{%- endif %}
ports:
- containerPort: 3000
name: external-http
- containerPort: 3001
name: internal-http
- containerPort: 4000
name: external-grpc
- containerPort: 4001
name: internal-grpc

View File

@ -1,42 +0,0 @@
{%- set service_name = vars.cuddle_vars.service -%}
{%- set cluster_name = vars.cluster_vars.name -%}
{%- set cluster_namespace = vars.cluster_vars.namespace -%}
apiVersion: v1
kind: Service
metadata:
name: "{{ service_name }}"
namespace: {{ cluster_namespace }}
spec:
selector:
app: {{ service_name }}
cluster: {{ cluster_name }}
type: LoadBalancer
ports:
- name: external-http
port: 3000
targetPort: 3000
- name: internal-http
port: 3001
targetPort: 3001
---
apiVersion: v1
kind: Service
metadata:
name: "{{ service_name }}-grpc"
namespace: {{ cluster_namespace }}
annotations:
traefik.ingress.kubernetes.io/service.serversscheme: h2c
spec:
selector:
app: {{ service_name }}
cluster: {{ cluster_name }}
type: LoadBalancer
ports:
- name: external-grpc
port: 4000
targetPort: 4000
- name: internal-grpc
port: 4001
targetPort: 4001

View File

@ -1,134 +0,0 @@
kind: pipeline
name: cuddle-rust-service-plan
type: docker
steps:
- name: load_secret
image: debian:buster-slim
volumes:
- name: ssh
path: /root/.ssh/
environment:
SSH_KEY:
from_secret: gitea_id_ed25519
commands:
- mkdir -p $HOME/.ssh/
- echo "$SSH_KEY" | base64 -d > $HOME/.ssh/id_ed25519
- chmod -R 600 ~/.ssh
- |
cat >$HOME/.ssh/config <<EOL
Host git.front.kjuulh.io
IdentityFile $HOME/.ssh/id_ed25519
IdentitiesOnly yes
UserKnownHostsFile=/dev/null
StrictHostKeyChecking no
EOL
- chmod 700 ~/.ssh/config
- name: build pr
image: kasperhermansen/{{bin_name}}:{{image_tag}}
volumes:
- name: ssh
path: /root/.ssh/
commands:
- eval `ssh-agent`
- ssh-add
- echo "$DOCKER_PASSWORD" | docker login --password-stdin --username="$DOCKER_USERNAME" docker.io
- export CLUSTER=clank-dev
- cuddle --version
- {{ bin_name }} pr
environment:
DAGGER_CLOUD_TOKEN:
from_secret: dagger_cloud_token
DRONE_HOST: "https://ci.i.kjuulh.io"
DRONE_USER: "kjuulh"
DRONE_TOKEN:
from_secret: drone_token
REGISTRY_CACHE_USERNAME:
from_secret: registry_cache_username
REGISTRY_CACHE_PASSWORD:
from_secret: registry_cache_password
REGISTRY_CACHE_TOKEN:
from_secret: registry_cache_token
REGISTRY_CACHE_url:
from_secret: registry_cache_url
DOCKER_BUILDKIT: 1
DOCKER_PASSWORD:
from_secret: docker_password
DOCKER_USERNAME:
from_secret: docker_username
CUDDLE_SECRETS_PROVIDER: 1password
CUDDLE_ONE_PASSWORD_DOT_ENV: ".env.ci"
CUDDLE_SSH_AGENT: "true"
GIT_PASSWORD:
from_secret: git_password
CI_PREFIX: "/mnt/ci/ci"
DOCKER_HOST: "tcp://192.168.1.155:2376"
CUDDLE_PLEASE_TOKEN:
from_secret: cuddle_please_token
OP_SERVICE_ACCOUNT_TOKEN:
from_secret: op_service_account_token
when:
event:
- pull_request
depends_on:
- "load_secret"
- name: build main
image: kasperhermansen/{{bin_name}}:{{image_tag}}
volumes:
- name: ssh
path: /root/.ssh/
commands:
- eval `ssh-agent`
- ssh-add
- echo "$DOCKER_PASSWORD" | docker login --password-stdin --username="$DOCKER_USERNAME" docker.io
- export CLUSTER=clank-prod
- cuddle --version
- {{ bin_name }} main
environment:
DAGGER_CLOUD_TOKEN:
from_secret: dagger_cloud_token
DRONE_HOST: "https://ci.i.kjuulh.io"
DRONE_USER: "kjuulh"
DRONE_TOKEN:
from_secret: drone_token
REGISTRY_CACHE_USERNAME:
from_secret: registry_cache_username
REGISTRY_CACHE_PASSWORD:
from_secret: registry_cache_password
REGISTRY_CACHE_TOKEN:
from_secret: registry_cache_token
REGISTRY_CACHE_url:
from_secret: registry_cache_url
DOCKER_BUILDKIT: 1
DOCKER_PASSWORD:
from_secret: docker_password
DOCKER_USERNAME:
from_secret: docker_username
CUDDLE_SECRETS_PROVIDER: 1password
CUDDLE_ONE_PASSWORD_DOT_ENV: ".env.ci"
CUDDLE_SSH_AGENT: "true"
GIT_PASSWORD:
from_secret: git_password
CI_PREFIX: "/mnt/ci/ci"
DOCKER_HOST: "tcp://192.168.1.155:2376"
CUDDLE_PLEASE_TOKEN:
from_secret: cuddle_please_token
OP_SERVICE_ACCOUNT_TOKEN:
from_secret: op_service_account_token
when:
event:
- push
branch:
- main
- master
exclude:
- pull_request
depends_on:
- "load_secret"
volumes:
- name: ssh
temp: {}

View File

@ -16,7 +16,5 @@ spec:
name: external_http
- containerPort: 3001
name: internal_http
- containerPort: 4000
name: external_grpc
- containerPort: 4001
- containerPort: 3002
name: internal_grpc

View File

@ -0,0 +1,9 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ service | lower | replace(from='_', to='-') }}-config"
data:
AWS_ACCESS_KEY_ID: "dNo8Kxbg08WcWVJdazXZ"
AWS_SECRET_ACCESS_KEY: "5NUnyOZMvIu89FJbqKwkynSvJVPRc5lsprsqm0c8"
AWS_ENDPOINT_URL: "https://api.minio.i.kjuulh.io"

View File

@ -0,0 +1,7 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ service | lower | replace(from='_', to='-') }}-database-crdb"
data:
# DATABASE_URL: "postgresql://{{ service | lower | replace(from='-', to='_') }}:strongpassword123@prod-cluster:26257/{{ service | lower | replace(from='-', to='_') }}"
DATABASE_URL: "postgresql://root@prod-cluster:26257/defaultdb"

View File

@ -1,5 +1,3 @@
{% set_global cluster_vars = filter_by_prefix(prefix=["clusters", cluster]) %}
apiVersion: apps/v1
kind: Deployment
metadata:
@ -22,18 +20,10 @@ spec:
name: internal-http
- containerPort: 3002
name: internal-grpc
{% if cluster_vars.config or cluster_vars.secrets %}
env:
{% for secret in cluster_vars.secrets %}
- name: SECRET_USERNAME
valueFrom:
secretKeyRef:
name: my-secret
key: username
- name: SECRET_PASSWORD
valueFrom:
secretKeyRef:
name: my-secret
key: password
{% endfor %}
{% endif %}
envFrom:
- configMapRef:
name: "{{ service | lower | replace(from='_', to='-') }}-database-crdb"
- configMapRef:
name: "{{ service | lower | replace(from='_', to='-') }}-config"
- configMapRef:
name: "{{ service | lower | replace(from='_', to='-') }}-message-queue-nats"

View File

@ -1,3 +1,4 @@
{% set_global cluster_vars = filter_by_prefix(prefix=["clusters", cluster]) %}
apiVersion: kustomize.config.k8s.io/v1beta1
@ -6,6 +7,9 @@ kind: Kustomization
resources:
- deployment.yaml
- service.yaml
- crdb.configmap.yaml
- nats.configmap.yaml
- configmap.yaml
commonLabels:
app: "{{ service }}"

View File

@ -0,0 +1,8 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ service | lower | replace(from='_', to='-') }}-message-queue-nats"
data:
NATS_USERNAME: "{{ service | lower | replace(from='_', to='-') }}"
NATS_PASSWORD: "{{ service | lower | replace(from='_', to='-') }}"
NATS_URL: "nats"