kjuulh/cuddle-v2
1
0
Fork 0
Code Issues 1 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: add cleanup for rust builder

Browse Source 
Signed-off-by: kjuulh <contact@kjuulh.io>
This commit is contained in:
Kasper Juul Hermansen 2024-10-26 13:49:46 +02:00
parent 7804eaa667
commit 37bf97972b
Signed by: kjuulh
GPG Key ID: D85D7535F18F35FA
3 changed files with 227 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

203
crates/cuddle/src/actions.rs
View File

@ -1,6 +1,6 @@
use std::{
collections::BTreeMap,
future::{self, Future},
future::Future,
ops::Deref,
path::{Path, PathBuf},
pin::Pin,
@ -14,155 +14,7 @@ use crate::state::validated_project::Value;
pub mod builder;
pub mod rust_builder {
use std::{
env::temp_dir,
path::{Path, PathBuf},
};
use serde::Deserialize;
use crate::actions::CuddleActionsSchema;
use super::ExecutableActions;
pub struct RustActionsBuilder {
root_path: PathBuf,
}
impl RustActionsBuilder {
pub fn new(root_path: PathBuf) -> Self {
Self { root_path }
}
fn actions_path(&self) -> Option<PathBuf> {
let actions_path = self.root_path.join("actions/rust");
if !actions_path.exists() {
return None;
}
Some(actions_path)
}
fn global_registry(&self) -> anyhow::Result<Option<PathBuf>> {
if let Some(dir) = dirs::cache_dir().map(|c| c.join("sh.cuddle/registry/actions/rust"))
{
if !dir.exists() {
std::fs::create_dir_all(&dir)?;
}
Ok(Some(dir))
} else {
Ok(None)
}
}
pub async fn build(&self) -> anyhow::Result<ExecutableActions> {
tracing::debug!("building rust action: {}", self.root_path.display());
let Some(path) = self.actions_path() else {
anyhow::bail!(
"action was not found: {}",
self.root_path.display().to_string()
);
};
let actions_registry = self
.global_registry()?
.ok_or(anyhow::anyhow!("failed to find global registry"))?;
let staging_id = uuid::Uuid::new_v4();
let actions_temp = temp_dir()
.join("cuddle/actions")
.join(staging_id.to_string());
std::fs::create_dir_all(&actions_temp)?;
let mut hasher = blake3::Hasher::new();
tracing::debug!("moving file into: {}", actions_temp.display());
for entry in walkdir::WalkDir::new(&path) {
let entry = entry?;
let full_path = entry.path();
let rel_path = full_path
.strip_prefix(path.canonicalize()?.to_string_lossy().to_string())?
.to_string_lossy();
if rel_path.contains("target/")
|| rel_path.contains(".cuddle/")
|| rel_path.contains(".git/")
{
continue;
}
let metadata = entry.metadata()?;
if metadata.is_file() {
let temp_file_path = actions_temp.join(rel_path.to_string());
if let Some(temp_parent) = temp_file_path.parent() {
std::fs::create_dir_all(temp_parent)?;
}
std::fs::copy(entry.path(), temp_file_path)?;
hasher.update(entry.path().to_string_lossy().as_bytes());
let file_bytes = tokio::fs::read(entry.path()).await?;
hasher.update(&file_bytes);
}
}
let digest = hasher.finalize().to_hex().to_string();
let action_index = actions_registry.join(digest);
if action_index.exists() {
tracing::debug!("action already exists in: {}", action_index.display());
return self.get_actions(&action_index.join("action")).await;
}
std::fs::create_dir_all(&action_index)?;
tracing::debug!("building rust code: {}", actions_temp.display());
let mut cmd = tokio::process::Command::new("cargo");
let output = cmd
.args(vec!["build", "--release"])
.current_dir(&actions_temp)
.output()
.await?;
if !output.status.success() {
anyhow::bail!(
"cargo build failed: {}",
std::str::from_utf8(&output.stderr)?
);
}
let temp_file_bin_path = actions_temp.join("target/release/action");
tokio::fs::copy(temp_file_bin_path, action_index.join("action")).await?;
self.get_actions(&action_index.join("action")).await
}
pub async fn get_actions(&self, action_path: &Path) -> anyhow::Result<ExecutableActions> {
tracing::debug!("querying schema: {}", action_path.display());
let output = tokio::process::Command::new(action_path.to_string_lossy().to_string())
.arg("schema")
.output()
.await?;
let actions: CuddleActionsSchema = match serde_json::from_slice(&output.stdout) {
Ok(output) => output,
Err(e) => {
let schema_output = std::str::from_utf8(&output.stdout)?;
anyhow::bail!(
"failed to query schema: {} {}",
e.to_string(),
schema_output
)
}
};
Ok(actions.to_executable(action_path)?)
}
}
}
pub mod rust_builder;
pub struct Actions {
variants: Vec<ActionVariant>,
@ -191,6 +43,8 @@ impl Actions {
pub async fn build(&mut self) -> anyhow::Result<ExecutableActions> {
let mut executable_actions = Vec::default();
self.clean_cache().await?;
for variant in &mut self.variants {
match variant {
ActionVariant::Rust { root_path } => {
@ -209,6 +63,48 @@ impl Actions {
Ok(exec_actions)
}
fn global_registry(&self) -> anyhow::Result<Option<PathBuf>> {
if let Some(dir) = dirs::cache_dir().map(|c| c.join("sh.cuddle/registry/actions/rust")) {
if !dir.exists() {
std::fs::create_dir_all(&dir)?;
}
Ok(Some(dir))
} else {
Ok(None)
}
}
/// clean_cache checks whether a given function has been used in the last month, if it hasn't it is automatically removed, and potentially reconstructed again on demand
pub async fn clean_cache(&mut self) -> anyhow::Result<()> {
let now = std::time::SystemTime::now();
let mut file_stream = tokio::fs::read_dir(
self.global_registry()?
.ok_or(anyhow::anyhow!("failed to get global registry"))?,
)
.await?;
while let Ok(Some(entry)) = file_stream.next_entry().await {
tracing::trace!("checking file: {}", entry.path().display());
let metadata = entry.metadata().await?;
if metadata.is_dir() {
let modified = metadata.modified()?;
let cache_threshold = now
.checked_sub(std::time::Duration::from_secs(60 * 24 * 30)) // Cache duration is a month
.expect("to be able to have a systemtime above a week");
if modified < cache_threshold {
tracing::debug!("cleaning up entry: {}", entry.path().display());
tokio::fs::remove_dir_all(entry.path()).await?;
}
}
}
Ok(())
}
}
#[derive(Debug, Deserialize)]
@ -272,6 +168,13 @@ impl CuddleActionsSchema {
let action_path = action_path.clone();
Box::pin(async move {
if let Some(parent) = PathBuf::from(&action_path).parent() {
tokio::process::Command::new("touch")
.arg(parent)
.output()
.await?;
}
tracing::debug!("calling: {}", name);
let mut cmd = tokio::process::Command::new(action_path);
cmd.args(["do", &name]);

2
crates/cuddle/src/actions/builder.rs
View File

@ -1 +1 @@
pub struct ActionsBuilder {}

173
crates/cuddle/src/actions/rust_builder.rs Normal file
View File

@ -0,0 +1,173 @@
use std::{
env::temp_dir,
path::{Path, PathBuf},
};
use crate::actions::CuddleActionsSchema;
use super::ExecutableActions;
pub struct RustActionsBuilder {
root_path: PathBuf,
}
impl RustActionsBuilder {
pub fn new(root_path: PathBuf) -> Self {
Self { root_path }
}
fn actions_path(&self) -> Option<PathBuf> {
let actions_path = self.root_path.join("actions/rust");
if !actions_path.exists() {
return None;
}
Some(actions_path)
}
fn global_registry(&self) -> anyhow::Result<Option<PathBuf>> {
if let Some(dir) = dirs::cache_dir().map(|c| c.join("sh.cuddle/registry/actions/rust")) {
if !dir.exists() {
std::fs::create_dir_all(&dir)?;
}
Ok(Some(dir))
} else {
Ok(None)
}
}
pub async fn build(&self) -> anyhow::Result<ExecutableActions> {
tracing::debug!("building rust action: {}", self.root_path.display());
let Some(path) = self.actions_path() else {
anyhow::bail!(
"action was not found: {}",
self.root_path.display().to_string()
);
};
let actions_registry = self
.global_registry()?
.ok_or(anyhow::anyhow!("failed to find global registry"))?;
let staging_id = uuid::Uuid::new_v4();
let actions_temp = temp_dir()
.join("cuddle/actions")
.join(staging_id.to_string());
std::fs::create_dir_all(&actions_temp)?;
let actions_temp: TempGuard = actions_temp.into();
let mut hasher = blake3::Hasher::new();
tracing::debug!("moving file into: {}", actions_temp.display());
for entry in walkdir::WalkDir::new(&path) {
let entry = entry?;
let full_path = entry.path();
let rel_path = full_path
.strip_prefix(path.canonicalize()?.to_string_lossy().to_string())?
.to_string_lossy();
if rel_path.contains("target/")
|| rel_path.contains(".cuddle/")
|| rel_path.contains(".git/")
{
continue;
}
let metadata = entry.metadata()?;
if metadata.is_file() {
let temp_file_path = actions_temp.join(rel_path.to_string());
if let Some(temp_parent) = temp_file_path.parent() {
std::fs::create_dir_all(temp_parent)?;
}
std::fs::copy(entry.path(), temp_file_path)?;
hasher.update(rel_path.as_bytes());
let file_bytes = tokio::fs::read(entry.path()).await?;
hasher.update(&file_bytes);
}
}
let digest = hasher.finalize().to_hex().to_string();
let action_index = actions_registry.join(digest);
if action_index.exists() {
tracing::debug!("action already exists in: {}", action_index.display());
return self.get_actions(&action_index.join("action")).await;
}
std::fs::create_dir_all(&action_index)?;
tracing::debug!("building rust code: {}", actions_temp.display());
let mut cmd = tokio::process::Command::new("cargo");
let output = cmd
.args(vec!["build", "--release"])
.current_dir(actions_temp.as_path())
.output()
.await?;
if !output.status.success() {
anyhow::bail!(
"cargo build failed: {}",
std::str::from_utf8(&output.stderr)?
);
}
let temp_file_bin_path = actions_temp.join("target/release/action");
tokio::fs::copy(temp_file_bin_path, action_index.join("action")).await?;
self.get_actions(&action_index.join("action")).await
}
pub async fn get_actions(&self, action_path: &Path) -> anyhow::Result<ExecutableActions> {
tracing::debug!("querying schema: {}", action_path.display());
let output = tokio::process::Command::new(action_path.to_string_lossy().to_string())
.arg("schema")
.output()
.await?;
let actions: CuddleActionsSchema = match serde_json::from_slice(&output.stdout) {
Ok(output) => output,
Err(e) => {
let schema_output = std::str::from_utf8(&output.stdout)?;
anyhow::bail!(
"failed to query schema: {} {}",
e.to_string(),
schema_output
)
}
};
actions.to_executable(action_path)
}
}
struct TempGuard(PathBuf);
impl From<PathBuf> for TempGuard {
fn from(value: PathBuf) -> Self {
Self(value)
}
}
impl std::ops::Deref for TempGuard {
type Target = PathBuf;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl Drop for TempGuard {
fn drop(&mut self) {
match std::fs::remove_dir_all(&self.0) {
Ok(_) => {
tracing::trace!("cleaned up temp dir: {}", self.0.display());
}
Err(e) => panic!("{}", e),
}
}
}