feat: added benchmark

Cargo.lock

@@ -2,36 +2,30 @@
 # It is not intended for manual editing.
 version = 3
 
-[[package]]
-name = "addr2line"
-version = "0.19.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97"
-dependencies = [
- "gimli",
-]
-
 [[package]]
 name = "adler"
 version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
-[[package]]
-name = "anyhow"
-version = "1.0.71"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
-dependencies = [
- "backtrace",
-]
-
 [[package]]
 name = "ascii"
 version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e"
 
+[[package]]
+name = "async-scoped"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e7a6a57c8aeb40da1ec037f5d455836852f7a57e69e1b1ad3d8f38ac1d6cadf"
+dependencies = [
+ "futures",
+ "pin-project",
+ "slab",
+ "tokio",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.68"
@@ -49,21 +43,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
-[[package]]
-name = "backtrace"
-version = "0.3.67"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca"
-dependencies = [
- "addr2line",
- "cc",
- "cfg-if",
- "libc",
- "miniz_oxide 0.6.2",
- "object",
- "rustc-demangle",
-]
-
 [[package]]
 name = "base64"
 version = "0.21.0"
@@ -186,9 +165,13 @@ dependencies = [
 name = "dagger-runtime-benchmark"
 version = "0.1.0"
 dependencies = [
- "anyhow",
+ "async-scoped",
  "dagger-sdk",
+ "eyre",
+ "stopwatch",
  "tokio",
+ "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
@@ -378,7 +361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743"
 dependencies = [
  "crc32fast",
- "miniz_oxide 0.7.1",
+ "miniz_oxide",
 ]
 
 [[package]]
@@ -396,6 +379,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fuchsia-cprng"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
+
 [[package]]
 name = "futures"
 version = "0.3.28"
@@ -506,12 +495,6 @@ dependencies = [
  "wasi",
 ]
 
-[[package]]
-name = "gimli"
-version = "0.27.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4"
-
 [[package]]
 name = "graphql-introspection-query"
 version = "0.2.0"
@@ -822,15 +805,6 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
-[[package]]
-name = "miniz_oxide"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"
-dependencies = [
- "adler",
-]
-
 [[package]]
 name = "miniz_oxide"
 version = "0.7.1"
@@ -862,6 +836,84 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "num"
+version = "0.1.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e"
+dependencies = [
+ "num-bigint",
+ "num-complex",
+ "num-integer",
+ "num-iter",
+ "num-rational",
+ "num-traits",
+]
+
+[[package]]
+name = "num-bigint"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e63899ad0da84ce718c14936262a41cee2c79c981fc0a0e7c7beb47d5a07e8c1"
+dependencies = [
+ "num-integer",
+ "num-traits",
+ "rand",
+ "rustc-serialize",
+]
+
+[[package]]
+name = "num-complex"
+version = "0.1.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b288631d7878aaf59442cffd36910ea604ecd7745c36054328595114001c9656"
+dependencies = [
+ "num-traits",
+ "rustc-serialize",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
+dependencies = [
+ "autocfg",
+ "num-traits",
+]
+
+[[package]]
+name = "num-iter"
+version = "0.1.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.1.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee314c74bd753fc86b4780aa9475da469155f3848473a261d2d18e35245a784e"
+dependencies = [
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+ "rustc-serialize",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "num_cpus"
 version = "1.15.0"
@@ -872,15 +924,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "object"
-version = "0.30.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "once_cell"
 version = "1.17.1"
@@ -922,6 +965,26 @@ version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
 
+[[package]]
+name = "pin-project"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.16",
+]
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.9"
@@ -962,6 +1025,43 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rand"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
+dependencies = [
+ "fuchsia-cprng",
+ "libc",
+ "rand_core 0.3.1",
+ "rdrand",
+ "winapi",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
+dependencies = [
+ "rand_core 0.4.2",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
+
+[[package]]
+name = "rdrand"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"
@@ -1048,10 +1148,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "rustc-demangle"
+name = "rustc-serialize"
-version = "0.1.23"
+version = "0.3.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
+checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
 
 [[package]]
 name = "rustix"
@@ -1223,6 +1323,15 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 
+[[package]]
+name = "stopwatch"
+version = "0.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d04b5ebc78da44d3a456319d8bc2783e7d8cc7ccbb5cb4dc3f54afbd93bf728"
+dependencies = [
+ "num",
+]
+
 [[package]]
 name = "strsim"
 version = "0.10.0"

Cargo.toml

@@ -6,6 +6,10 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-anyhow = { version = "1.0.71", features = ["backtrace"] }
+eyre = { version = "*" }
 dagger-sdk = "0.2.22"
 tokio = { version = "1.28.1", features = ["full"] }
+async-scoped = { version = "0.7.1", features = ["use-tokio"] }
+tracing-subscriber = "0.3.17"
+tracing = { version = "0.1.37", features = ["log"] }
+stopwatch = "0.0.7"

actions/upload.go

@@ -14,7 +14,7 @@ func Upload(ctx context.Context) error {
 	}
 
 	src := client.Host().
-		Directory("bench_app", dagger.HostDirectoryOpts{Exclude: []string{"target/"}})
+		Directory("bench_app/", dagger.HostDirectoryOpts{Exclude: []string{"target/"}})
 
 	rust_base := client.Container().
 		From("rustlang/rust:nightly").
@@ -24,9 +24,11 @@ func Upload(ctx context.Context) error {
 
 	rust_binary := rust_base.File("target/release/bench_app")
 
-	rust_application_image := client.Container().
+	rust_application_image := client.
-		From("alpine").
+		Container().
-		WithFile("/usr/bin/bench_app", rust_binary)
+		From("debian:bullseye-slim").
+		WithFile("/usr/bin/bench_app", rust_binary).
+		WithExec([]string{"/usr/bin/bench_app", "-h"})
 
 	wasm_binary := client.Container().
 		From("rustlang/rust:nightly").
@@ -46,10 +48,14 @@ func Upload(ctx context.Context) error {
 		return err
 	}
 
-	_, err = rust_application_image.Export(ctx, "dist/bench_app.tar.gz")
+	imagetag, err := rust_application_image.Publish(
+		ctx,
+		"kasperhermansen/dagger-runtime-benchmark:latest",
+	)
 	if err != nil {
 		return err
 	}
+	println(imagetag)
 
 	return nil
 }

shuttle.yaml

@@ -1,3 +1,10 @@
 plan: false
 vars:
   service: dagger-runtime-benchmark
+
+
+scripts:
+  clear_cache:
+    actions:
+      - shell: docker stop $(docker ps -a | grep dagger-engine | awk '{print $1}') || true
+      - shell: docker rm $(docker ps -a | grep dagger-engine | awk '{print $1}') || true

src/main.rs

@@ -1,4 +1,184 @@
+use stopwatch::Stopwatch;
+use tokio::io::AsyncWriteExt;
+use tokio::process;
+
 #[tokio::main]
-async fn main() -> anyhow::Result<()> {
+async fn main() -> eyre::Result<()> {
+    tracing_subscriber::fmt().pretty().init();
+
+    let client = dagger_sdk::connect().await?;
+
+    let (tx, mut rx) = tokio::sync::mpsc::channel(4);
+
+    async_scoped::TokioScope::scope_and_block(|s| {
+        let dclient = client.clone();
+        let dtx = tx.clone();
+        s.spawn(async move {
+            let dclient = dclient.pipeline("build-image");
+            tracing::info!("building 1. complete image");
+
+            let sw = Stopwatch::start_new();
+            let src = dclient
+                .host()
+                .directory_opts(
+                    "bench_app",
+                    dagger_sdk::HostDirectoryOptsBuilder::default()
+                        .exclude(vec!["target/"])
+                        .build()
+                        .unwrap(),
+                )
+                .id()
+                .await
+                .unwrap();
+
+            let _ = dclient
+                .container()
+                .from("rustlang/rust:nightly")
+                .with_directory("/mnt/app", src)
+                .with_workdir("/mnt/app")
+                .with_exec(vec!["cargo", "build", "--release"])
+                .with_exec(vec!["target/release/bench_app", "bench"])
+                .exit_code()
+                .await
+                .unwrap();
+
+            dtx.send(format!(
+                "1. build-image: {}",
+                sw.elapsed_ms() as f64 / 1000.0
+            ))
+            .await
+            .unwrap();
+        });
+
+        let dclient = client.clone();
+        let dtx = tx.clone();
+        s.spawn(async move {
+            let dclient = dclient.pipeline("pull-image");
+            tracing::info!("building 2. pull image");
+            let sw = Stopwatch::start_new();
+
+            if let Err(e) = process::Command::new("docker")
+                .args(&[
+                    "image",
+                    "rm",
+                    "kasperhermansen/dagger-runtime-benchmark:latest",
+                ])
+                .output()
+                .await
+            {
+                tracing::warn!(msg = e.to_string(), "failed to delete image");
+            }
+
+            let _ = dclient
+                .container()
+                .from("kasperhermansen/dagger-runtime-benchmark:latest")
+                .with_exec(vec!["/usr/bin/bench_app", "bench"])
+                .exit_code()
+                .await
+                .unwrap();
+            dtx.send(format!(
+                "2. pull-image: {}",
+                sw.elapsed_ms() as f64 / 1000.0
+            ))
+            .await
+            .unwrap();
+        });
+
+        let dclient = client.clone();
+        let dtx = tx.clone();
+        s.spawn(async move {
+            let dclient = dclient.pipeline("run-wasm");
+            tracing::info!("building 3. pull wasm");
+            let sw = Stopwatch::start_new();
+
+            let src = dclient
+                .host()
+                .directory(".")
+                .file("dist/wasm/bench_app.wasm")
+                .id()
+                .await
+                .unwrap();
+
+            if let Err(e) = process::Command::new("docker")
+                .args(&["image", "rm", "kasperhermansen/dagger-wasmer:latest"])
+                .output()
+                .await
+            {
+                tracing::warn!(msg = e.to_string(), "failed to delete image");
+            }
+
+            let _ = dclient
+                .container()
+                .from("kasperhermansen/dagger-wasmer:latest")
+                .with_file("/usr/bin/bench_app.wasm", src)
+                .with_exec(vec!["wasmer", "run", "/usr/bin/bench_app.wasm", "bench"])
+                .exit_code()
+                .await
+                .unwrap();
+
+            dtx.send(format!("3. run-wasm: {}", sw.elapsed_ms() as f64 / 1000.0))
+                .await
+                .unwrap();
+        });
+
+        let dclient = client.clone();
+        let dtx = tx.clone();
+        s.spawn(async move {
+            let dclient = dclient.pipeline("run-binary");
+            tracing::info!("building 4. run binary");
+            let sw = Stopwatch::start_new();
+
+            let src = dclient
+                .host()
+                .directory(".")
+                .file("dist/binary/bench_app")
+                .id()
+                .await
+                .unwrap();
+
+            if let Err(e) = process::Command::new("docker")
+                .args(&["image", "rm", "debian:bullseye-slim"])
+                .output()
+                .await
+            {
+                tracing::warn!(msg = e.to_string(), "failed to delete image");
+            }
+
+            let _ = dclient
+                .container()
+                .from("debian:bullseye-slim")
+                .with_file("/usr/bin/bench_app", src)
+                .with_exec(vec!["/usr/bin/bench_app", "bench"])
+                .exit_code()
+                .await
+                .unwrap();
+
+            dtx.send(format!(
+                "4. run-binary: {}",
+                sw.elapsed_ms() as f64 / 1000.0
+            ))
+            .await
+            .unwrap();
+        });
+    });
+
+    drop(tx);
+
+    let mut file = tokio::fs::File::create("reports/timings.md").await?;
+    file.write_all(b"# Dagger runtime benchmark results\n")
+        .await?;
+
+    let mut results = Vec::new();
+    while let Some(res) = rx.recv().await {
+        results.push(res);
+    }
+    results.sort();
+
+    for res in results {
+        file.write_all(format!("{}\n", res).as_bytes()).await?;
+    }
+
+    file.sync_all().await?;
+
     Ok(())
 }