feat: refactor projects

Signed-off-by: kjuulh <contact@kjuulh.io>
This commit is contained in:
Kasper Juul Hermansen 2024-10-26 22:27:16 +02:00
parent 350a3669b0
commit 6729f6e794
Signed by: kjuulh
GPG Key ID: D85D7535F18F35FA
20 changed files with 614 additions and 220 deletions

68
Cargo.lock generated
View File

@ -245,6 +245,32 @@ dependencies = [
"typenum",
]
[[package]]
name = "cuddle-actions"
version = "0.2.0"
dependencies = [
"anyhow",
"blake3",
"cuddle-actions-api",
"cuddle-rust-actions",
"cuddle-value",
"dirs",
"serde",
"sha256",
"tokio",
"tracing",
"walkdir",
]
[[package]]
name = "cuddle-actions-api"
version = "0.2.0"
dependencies = [
"anyhow",
"cuddle-lazy",
"serde",
]
[[package]]
name = "cuddle-actions-sdk"
version = "0.2.0"
@ -259,7 +285,7 @@ dependencies = [
[[package]]
name = "cuddle-actions-sdk"
version = "0.2.0"
source = "git+ssh://git@git.front.kjuulh.io/kjuulh/cuddle-v2#6c4a05e43d668f2f8e5dc69f8eef8218c9febf3e"
source = "git+ssh://git@git.front.kjuulh.io/kjuulh/cuddle-v2#350a3669b04e3ae37fbb31f842f69e1e8afa1721"
dependencies = [
"anyhow",
"clap",
@ -267,6 +293,35 @@ dependencies = [
"serde_json",
]
[[package]]
name = "cuddle-file"
version = "0.2.0"
[[package]]
name = "cuddle-lazy"
version = "0.2.0"
dependencies = [
"anyhow",
]
[[package]]
name = "cuddle-rust-actions"
version = "0.2.0"
dependencies = [
"anyhow",
"blake3",
"cuddle-actions-api",
"cuddle-lazy",
"dirs",
"serde",
"serde_json",
"sha256",
"tokio",
"tracing",
"uuid",
"walkdir",
]
[[package]]
name = "cuddle-v2"
version = "0.2.0"
@ -274,6 +329,9 @@ dependencies = [
"anyhow",
"blake3",
"clap",
"cuddle-actions",
"cuddle-actions-api",
"cuddle-value",
"dirs",
"dotenv",
"fs_extra",
@ -288,6 +346,14 @@ dependencies = [
"walkdir",
]
[[package]]
name = "cuddle-value"
version = "0.2.0"
dependencies = [
"serde",
"toml",
]
[[package]]
name = "diff"
version = "0.1.13"

View File

@ -7,6 +7,11 @@ version = "0.2.0"
[workspace.dependencies]
cuddle = { path = "crates/cuddle" }
cuddle-actions = { path = "crates/cuddle-actions" }
cuddle-rust-actions = { path = "crates/cuddle-rust-actions" }
cuddle-lazy = { path = "crates/cuddle-lazy" }
cuddle-actions-api = { path = "crates/cuddle-actions-api" }
cuddle-value = { path = "crates/cuddle-value" }
anyhow = { version = "1" }
tokio = { version = "1", features = ["full"] }
@ -17,3 +22,9 @@ dotenv = { version = "0.15" }
serde = { version = "1.0.197", features = ["derive"] }
serde_json = "1.0.127"
uuid = { version = "1.7.0", features = ["v4"] }
toml = "0.8.19"
dirs = "5.0.1"
walkdir = "2.5.0"
sha256 = "1.5.0"
blake3 = "1.5.4"
fs_extra = "1.3.0"

View File

@ -0,0 +1,9 @@
[package]
name = "cuddle-actions-api"
edition = "2021"
version.workspace = true
[dependencies]
cuddle-lazy.workspace = true
serde.workspace = true
anyhow.workspace = true

View File

@ -0,0 +1,30 @@
use std::collections::BTreeMap;
use cuddle_lazy::LazyResolve;
use serde::{Deserialize, Serialize};
#[derive(Default)]
pub struct ExecutableActions {
pub actions: Vec<ExecutableAction>,
}
pub struct ExecutableAction {
pub name: String,
pub description: String,
pub flags: BTreeMap<String, ExecutableActionFlag>,
pub call_fn: LazyResolve,
}
impl ExecutableAction {
pub async fn call(&self) -> anyhow::Result<()> {
self.call_fn.call().await?;
Ok(())
}
}
#[derive(Debug, Serialize, Deserialize)]
pub enum ExecutableActionFlag {
String,
Bool,
}

View File

@ -0,0 +1,18 @@
[package]
name = "cuddle-actions"
edition = "2021"
version.workspace = true
[dependencies]
cuddle-rust-actions.workspace = true
cuddle-actions-api.workspace = true
cuddle-value.workspace = true
serde.workspace = true
tokio.workspace = true
tracing.workspace = true
anyhow.workspace = true
dirs.workspace = true
walkdir.workspace = true
sha256.workspace = true
blake3.workspace = true

View File

@ -0,0 +1,101 @@
use std::path::{Path, PathBuf};
use cuddle_actions_api::ExecutableActions;
use cuddle_rust_actions::RustActionsBuilder;
use cuddle_value::Value;
pub struct Actions {
variants: Vec<ActionVariant>,
}
impl Actions {
pub async fn new(path: &Path, value: &Value) -> anyhow::Result<Option<Self>> {
let Some(project) = value.get(&["project", "actions"]) else {
tracing::debug!("found no actions folder");
return Ok(None);
};
let mut variants = Vec::default();
if let Some(Value::Bool(true)) = project.get(&["rust"]) {
tracing::debug!("rust actions enabled");
variants.push(ActionVariant::Rust {
root_path: path.to_path_buf(),
});
} else {
tracing::trace!("rust actions not enabled");
}
Ok(Some(Self { variants }))
}
pub async fn build(&mut self) -> anyhow::Result<ExecutableActions> {
let mut executable_actions = Vec::default();
self.clean_cache().await?;
for variant in &mut self.variants {
match variant {
ActionVariant::Rust { root_path } => {
let actions = RustActionsBuilder::new(root_path.clone()).build().await?;
executable_actions.push(actions);
}
ActionVariant::Docker => todo!(),
}
}
let mut exec_actions = ExecutableActions::default();
for mut actions in executable_actions {
exec_actions.actions.append(&mut actions.actions);
}
Ok(exec_actions)
}
fn global_registry(&self) -> anyhow::Result<Option<PathBuf>> {
if let Some(dir) = dirs::cache_dir().map(|c| c.join("sh.cuddle/registry/actions/rust")) {
if !dir.exists() {
std::fs::create_dir_all(&dir)?;
}
Ok(Some(dir))
} else {
Ok(None)
}
}
/// clean_cache checks whether a given function has been used in the last month, if it hasn't it is automatically removed, and potentially reconstructed again on demand
pub async fn clean_cache(&mut self) -> anyhow::Result<()> {
let now = std::time::SystemTime::now();
let mut file_stream = tokio::fs::read_dir(
self.global_registry()?
.ok_or(anyhow::anyhow!("failed to get global registry"))?,
)
.await?;
while let Ok(Some(entry)) = file_stream.next_entry().await {
tracing::trace!("checking file: {}", entry.path().display());
let metadata = entry.metadata().await?;
if metadata.is_dir() {
let modified = metadata.modified()?;
let cache_threshold = now
.checked_sub(std::time::Duration::from_secs(60 * 24 * 30)) // Cache duration is a month
.expect("to be able to have a systemtime above a week");
if modified < cache_threshold {
tracing::debug!("cleaning up entry: {}", entry.path().display());
tokio::fs::remove_dir_all(entry.path()).await?;
}
}
}
Ok(())
}
}
pub enum ActionVariant {
Rust { root_path: PathBuf },
Docker,
}

View File

@ -0,0 +1,6 @@
[package]
name = "cuddle-file"
edition = "2021"
version.workspace = true
[dependencies]

View File

@ -0,0 +1,14 @@
pub fn add(left: u64, right: u64) -> u64 {
left + right
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn it_works() {
let result = add(2, 2);
assert_eq!(result, 4);
}
}

View File

@ -0,0 +1,7 @@
[package]
name = "cuddle-lazy"
edition = "2021"
version.workspace = true
[dependencies]
anyhow.workspace = true

View File

@ -0,0 +1,29 @@
use std::{future::Future, ops::Deref, pin::Pin, sync::Arc};
pub struct LazyResolve(
Arc<dyn Fn() -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send>> + Send + Sync>,
);
impl LazyResolve {
pub fn new(
func: Box<
dyn Fn() -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send>> + Send + Sync,
>,
) -> Self {
Self(Arc::new(func))
}
pub async fn call(&self) -> anyhow::Result<()> {
// Bad hack until .call becomes stable
(self.0)().await
}
}
impl Deref for LazyResolve {
type Target =
Arc<dyn Fn() -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send>> + Send + Sync>;
fn deref(&self) -> &Self::Target {
&self.0
}
}

View File

@ -0,0 +1,20 @@
[package]
name = "cuddle-rust-actions"
edition = "2021"
version.workspace = true
[dependencies]
cuddle-lazy.workspace = true
cuddle-actions-api.workspace = true
serde.workspace = true
tokio.workspace = true
tracing.workspace = true
anyhow.workspace = true
dirs.workspace = true
walkdir.workspace = true
sha256.workspace = true
blake3.workspace = true
uuid.workspace = true
serde_json.workspace = true

View File

@ -0,0 +1,233 @@
use std::{
collections::BTreeMap,
env::temp_dir,
path::{Path, PathBuf},
};
use cuddle_actions_api::{ExecutableAction, ExecutableActions};
use cuddle_lazy::LazyResolve;
use serde::Deserialize;
pub struct RustActionsBuilder {
root_path: PathBuf,
}
impl RustActionsBuilder {
pub fn new(root_path: PathBuf) -> Self {
Self { root_path }
}
fn actions_path(&self) -> Option<PathBuf> {
let actions_path = self.root_path.join("actions/rust");
if !actions_path.exists() {
return None;
}
Some(actions_path)
}
fn global_registry(&self) -> anyhow::Result<Option<PathBuf>> {
if let Some(dir) = dirs::cache_dir().map(|c| c.join("sh.cuddle/registry/actions/rust")) {
if !dir.exists() {
std::fs::create_dir_all(&dir)?;
}
Ok(Some(dir))
} else {
Ok(None)
}
}
pub async fn build(&self) -> anyhow::Result<ExecutableActions> {
tracing::debug!("building rust action: {}", self.root_path.display());
let Some(path) = self.actions_path() else {
anyhow::bail!(
"action was not found: {}",
self.root_path.display().to_string()
);
};
let actions_registry = self
.global_registry()?
.ok_or(anyhow::anyhow!("failed to find global registry"))?;
let staging_id = uuid::Uuid::new_v4();
let actions_temp = temp_dir()
.join("cuddle/actions")
.join(staging_id.to_string());
std::fs::create_dir_all(&actions_temp)?;
let actions_temp: TempGuard = actions_temp.into();
let mut hasher = blake3::Hasher::new();
tracing::debug!("moving file into: {}", actions_temp.display());
for entry in walkdir::WalkDir::new(&path) {
let entry = entry?;
let full_path = entry.path();
let rel_path = full_path
.strip_prefix(path.canonicalize()?.to_string_lossy().to_string())?
.to_string_lossy();
if rel_path.contains("target/")
|| rel_path.contains(".cuddle/")
|| rel_path.contains(".git/")
{
continue;
}
let metadata = entry.metadata()?;
if metadata.is_file() {
let temp_file_path = actions_temp.join(rel_path.to_string());
if let Some(temp_parent) = temp_file_path.parent() {
std::fs::create_dir_all(temp_parent)?;
}
std::fs::copy(entry.path(), temp_file_path)?;
hasher.update(rel_path.as_bytes());
let file_bytes = tokio::fs::read(entry.path()).await?;
hasher.update(&file_bytes);
}
}
let digest = hasher.finalize().to_hex().to_string();
let action_index = actions_registry.join(digest);
if action_index.exists() {
tracing::debug!("action already exists in: {}", action_index.display());
return self.get_actions(&action_index.join("action")).await;
}
std::fs::create_dir_all(&action_index)?;
tracing::debug!("building rust code: {}", actions_temp.display());
let mut cmd = tokio::process::Command::new("cargo");
let output = cmd
.args(vec!["build", "--release"])
.current_dir(actions_temp.as_path())
.output()
.await?;
if !output.status.success() {
anyhow::bail!(
"cargo build failed: {}",
std::str::from_utf8(&output.stderr)?
);
}
let temp_file_bin_path = actions_temp.join("target/release/action");
tokio::fs::copy(temp_file_bin_path, action_index.join("action")).await?;
self.get_actions(&action_index.join("action")).await
}
pub async fn get_actions(&self, action_path: &Path) -> anyhow::Result<ExecutableActions> {
tracing::debug!("querying schema: {}", action_path.display());
let output = tokio::process::Command::new(action_path.to_string_lossy().to_string())
.arg("schema")
.output()
.await?;
let actions: CuddleActionsSchema = match serde_json::from_slice(&output.stdout) {
Ok(output) => output,
Err(e) => {
let schema_output = std::str::from_utf8(&output.stdout)?;
anyhow::bail!(
"failed to query schema: {} {}",
e.to_string(),
schema_output
)
}
};
actions.to_executable(action_path)
}
}
struct TempGuard(PathBuf);
impl From<PathBuf> for TempGuard {
fn from(value: PathBuf) -> Self {
Self(value)
}
}
impl std::ops::Deref for TempGuard {
type Target = PathBuf;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl Drop for TempGuard {
fn drop(&mut self) {
match std::fs::remove_dir_all(&self.0) {
Ok(_) => {
tracing::trace!("cleaned up temp dir: {}", self.0.display());
}
Err(e) => panic!("{}", e),
}
}
}
#[derive(Debug, Deserialize, Clone)]
struct CuddleActionsSchema {
actions: Vec<CuddleActionSchema>,
}
#[derive(Debug, Deserialize, Clone)]
struct CuddleActionSchema {
name: String,
}
impl CuddleActionsSchema {
fn to_executable(&self, action_path: &Path) -> anyhow::Result<ExecutableActions> {
Ok(ExecutableActions {
actions: self
.actions
.iter()
.cloned()
.map(|a| {
let name = a.name.clone();
let action_path = action_path.to_string_lossy().to_string();
ExecutableAction {
name: a.name,
description: String::new(),
flags: BTreeMap::default(),
call_fn: LazyResolve::new(Box::new(move || {
let name = name.clone();
let action_path = action_path.clone();
Box::pin(async move {
if let Some(parent) = PathBuf::from(&action_path).parent() {
tokio::process::Command::new("touch")
.arg(parent)
.output()
.await?;
}
tracing::debug!("calling: {}", name);
let mut cmd = tokio::process::Command::new(action_path);
cmd.args(["do", &name]);
let output = cmd.output().await?;
let stdout = std::str::from_utf8(&output.stdout)?;
for line in stdout.lines() {
println!("{}: {}", &name, line);
}
tracing::debug!("finished call for output: {}", &name);
Ok(())
})
})),
}
})
.collect(),
})
}
}

View File

@ -0,0 +1,8 @@
[package]
name = "cuddle-value"
edition = "2021"
version.workspace = true
[dependencies]
toml.workspace = true
serde.workspace = true

View File

@ -0,0 +1,43 @@
use std::collections::BTreeMap;
use serde::Serialize;
#[derive(Clone, PartialEq, Eq, Debug, Serialize)]
#[serde(untagged)]
pub enum Value {
String(String),
Bool(bool),
Array(Vec<Value>),
Map(BTreeMap<String, Value>),
}
impl From<&toml::Value> for Value {
fn from(value: &toml::Value) -> Self {
match value {
toml::Value::String(s) => Self::String(s.clone()),
toml::Value::Integer(i) => Self::String(i.to_string()),
toml::Value::Float(f) => Self::String(f.to_string()),
toml::Value::Boolean(b) => Self::Bool(*b),
toml::Value::Datetime(dt) => Self::String(dt.to_string()),
toml::Value::Array(array) => Self::Array(array.iter().map(|i| i.into()).collect()),
toml::Value::Table(tbl) => {
Self::Map(tbl.iter().map(|(k, v)| (k.clone(), v.into())).collect())
}
}
}
}
impl Value {
pub fn get(&self, path: &[&str]) -> Option<&Value> {
match path.split_first() {
Some((current, rest)) => match self {
Value::Map(map) => match map.get(&current.to_string()) {
Some(value) => value.get(rest),
None => None,
},
_ => None,
},
None => Some(self),
}
}
}

View File

@ -5,6 +5,10 @@ edition = "2021"
version.workspace = true
[dependencies]
cuddle-actions.workspace = true
cuddle-actions-api.workspace = true
cuddle-value.workspace = true
anyhow.workspace = true
tokio.workspace = true
tracing.workspace = true
@ -15,9 +19,9 @@ serde.workspace = true
serde_json.workspace = true
uuid.workspace = true
toml = "0.8.19"
fs_extra = "1.3.0"
dirs = "5.0.1"
walkdir = "2.5.0"
sha256 = "1.5.0"
blake3 = "1.5.4"
toml.workspace = true
dirs.workspace = true
fs_extra.workspace = true
walkdir.workspace = true
sha256.workspace = true
blake3.workspace = true

View File

@ -1,4 +1,4 @@
use cuddle_actions::AddActionOptions;
use cuddle_actions_sdk::AddActionOptions;
#[tokio::main]
async fn main() -> anyhow::Result<()> {

View File

@ -1,163 +1 @@
use std::{
collections::BTreeMap,
future::Future,
ops::Deref,
path::{Path, PathBuf},
pin::Pin,
sync::Arc,
};
use rust_builder::RustActionsBuilder;
use serde::{Deserialize, Serialize};
use crate::state::validated_project::Value;
pub mod builder;
pub mod rust_builder;
pub struct Actions {
variants: Vec<ActionVariant>,
}
impl Actions {
pub async fn new(path: &Path, value: &Value) -> anyhow::Result<Option<Self>> {
let Some(project) = value.get(&["project", "actions"]) else {
tracing::debug!("found no actions folder");
return Ok(None);
};
let mut variants = Vec::default();
if let Some(Value::Bool(true)) = project.get(&["rust"]) {
tracing::debug!("rust actions enabled");
variants.push(ActionVariant::Rust {
root_path: path.to_path_buf(),
});
} else {
tracing::trace!("rust actions not enabled");
}
Ok(Some(Self { variants }))
}
pub async fn build(&mut self) -> anyhow::Result<ExecutableActions> {
let mut executable_actions = Vec::default();
self.clean_cache().await?;
for variant in &mut self.variants {
match variant {
ActionVariant::Rust { root_path } => {
let actions = RustActionsBuilder::new(root_path.clone()).build().await?;
executable_actions.push(actions);
}
ActionVariant::Docker => todo!(),
}
}
let mut exec_actions = ExecutableActions::default();
for mut actions in executable_actions {
exec_actions.actions.append(&mut actions.actions);
}
Ok(exec_actions)
}
fn global_registry(&self) -> anyhow::Result<Option<PathBuf>> {
if let Some(dir) = dirs::cache_dir().map(|c| c.join("sh.cuddle/registry/actions/rust")) {
if !dir.exists() {
std::fs::create_dir_all(&dir)?;
}
Ok(Some(dir))
} else {
Ok(None)
}
}
/// clean_cache checks whether a given function has been used in the last month, if it hasn't it is automatically removed, and potentially reconstructed again on demand
pub async fn clean_cache(&mut self) -> anyhow::Result<()> {
let now = std::time::SystemTime::now();
let mut file_stream = tokio::fs::read_dir(
self.global_registry()?
.ok_or(anyhow::anyhow!("failed to get global registry"))?,
)
.await?;
while let Ok(Some(entry)) = file_stream.next_entry().await {
tracing::trace!("checking file: {}", entry.path().display());
let metadata = entry.metadata().await?;
if metadata.is_dir() {
let modified = metadata.modified()?;
let cache_threshold = now
.checked_sub(std::time::Duration::from_secs(60 * 24 * 30)) // Cache duration is a month
.expect("to be able to have a systemtime above a week");
if modified < cache_threshold {
tracing::debug!("cleaning up entry: {}", entry.path().display());
tokio::fs::remove_dir_all(entry.path()).await?;
}
}
}
Ok(())
}
}
pub enum ActionVariant {
Rust { root_path: PathBuf },
Docker,
}
#[derive(Default)]
pub struct ExecutableActions {
pub actions: Vec<ExecutableAction>,
}
pub struct ExecutableAction {
pub name: String,
pub description: String,
pub flags: BTreeMap<String, ExecutableActionFlag>,
call_fn: LazyResolve,
}
impl ExecutableAction {
pub async fn call(&self) -> anyhow::Result<()> {
// Bad hack until .call becomes stable
(self.call_fn.0)().await?;
Ok(())
}
}
#[derive(Debug, Serialize, Deserialize)]
pub enum ExecutableActionFlag {
String,
Bool,
}
struct LazyResolve(
Arc<dyn Fn() -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send>> + Send + Sync>,
);
impl LazyResolve {
pub fn new(
func: Box<
dyn Fn() -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send>> + Send + Sync,
>,
) -> Self {
Self(Arc::new(func))
}
}
impl Deref for LazyResolve {
type Target =
Arc<dyn Fn() -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send>> + Send + Sync>;
fn deref(&self) -> &Self::Target {
&self.0
}
}

View File

@ -1,9 +1,8 @@
use cuddle_value::Value;
use crate::{
cuddle_state::Cuddle,
state::{
validated_project::{Project, Value},
ValidatedState,
},
state::{validated_project::Project, ValidatedState},
};
pub struct GetCommand {

View File

@ -1,7 +1,8 @@
use cuddle_actions::Actions;
use cuddle_actions_api::ExecutableActions;
use validated_project::Project;
use crate::{
actions::{Actions, ExecutableActions},
plan::{self, ClonedPlan, PlanPathExt},
project::{self, ProjectPlan},
schema_validator::SchemaValidator,

View File

@ -1,10 +1,7 @@
use std::{
collections::BTreeMap,
path::{Path, PathBuf},
};
use std::path::{Path, PathBuf};
use anyhow::anyhow;
use serde::Serialize;
use cuddle_value::Value;
use toml::Table;
use crate::project::CUDDLE_PROJECT_FILE;
@ -48,43 +45,3 @@ impl Project {
Self::from_file(&cuddle_project_file, path)
}
}
#[derive(Clone, PartialEq, Eq, Debug, Serialize)]
#[serde(untagged)]
pub enum Value {
String(String),
Bool(bool),
Array(Vec<Value>),
Map(BTreeMap<String, Value>),
}
impl From<&toml::Value> for Value {
fn from(value: &toml::Value) -> Self {
match value {
toml::Value::String(s) => Self::String(s.clone()),
toml::Value::Integer(i) => Self::String(i.to_string()),
toml::Value::Float(f) => Self::String(f.to_string()),
toml::Value::Boolean(b) => Self::Bool(*b),
toml::Value::Datetime(dt) => Self::String(dt.to_string()),
toml::Value::Array(array) => Self::Array(array.iter().map(|i| i.into()).collect()),
toml::Value::Table(tbl) => {
Self::Map(tbl.iter().map(|(k, v)| (k.clone(), v.into())).collect())
}
}
}
}
impl Value {
pub fn get(&self, path: &[&str]) -> Option<&Value> {
match path.split_first() {
Some((current, rest)) => match self {
Value::Map(map) => match map.get(&current.to_string()) {
Some(value) => value.get(rest),
None => None,
},
_ => None,
},
None => Some(self),
}
}
}