diff --git a/cmd/dagger/cmd/do.go b/cmd/dagger/cmd/do.go index ec25d526..ff6f6a27 100644 --- a/cmd/dagger/cmd/do.go +++ b/cmd/dagger/cmd/do.go @@ -73,6 +73,8 @@ var doCmd = &cobra.Command{ <-doneCh + p.Context().TempDirs.Clean() + if err != nil { lg.Fatal().Err(err).Msg("failed to execute plan") } diff --git a/docs/drafts/1216-engine-load.md b/docs/drafts/1216-engine-load.md new file mode 100644 index 00000000..acfd41a1 --- /dev/null +++ b/docs/drafts/1216-engine-load.md @@ -0,0 +1,20 @@ +--- +slug: /1216/engine-load +displayed_sidebar: europa +--- + +# Loading a dagger image into a docker daemon + +Using `docker.#Load`, you can save a dagger image (`docker.#Image`) into a local or remote engine. + +It can be useful to debug or test a build locally before pushing. + +## Local daemon + +```cue file=./plans/local.cue +``` + +## Remote daemon, via SSH + +```cue file=./plans/ssh.cue +``` diff --git a/docs/drafts/plans/local.cue b/docs/drafts/plans/local.cue new file mode 100644 index 00000000..add0b372 --- /dev/null +++ b/docs/drafts/plans/local.cue @@ -0,0 +1,22 @@ +package main + +import ( + "dagger.io/dagger" + "universe.dagger.io/docker" +) + +dagger.#Plan & { + client: filesystem: "/var/run/docker.sock": read: contents: dagger.#Service + + actions: { + build: docker.#Build & { + ... + } + + load: docker.#Load & { + image: build.output + host: client.filesystem."/var/run/docker.sock".read.contents + tag: "myimage" + } + } +} diff --git a/docs/drafts/plans/ssh.cue b/docs/drafts/plans/ssh.cue new file mode 100644 index 00000000..bebeec9c --- /dev/null +++ b/docs/drafts/plans/ssh.cue @@ -0,0 +1,29 @@ +package main + +import ( + "dagger.io/dagger" + "universe.dagger.io/docker" +) + +dagger.#Plan & { + client: filesystem: { + "/home/user/.ssh/id_rsa": read: contents: dagger.#Secret + "/home/user/.ssh/known_hosts": read: contents: dagger.#Secret + } + + actions: { + build: docker.#Build & { + ... + } + + load: docker.#Load & { + image: build.output + tag: "myimage:v2" + host: "ssh://root@93.184.216.34" + ssh: { + key: client.filesystem."/home/user/.ssh/id_rsa".read.contents + knownHosts: client.filesystem."/home/user/.ssh/known_hosts".read.contents + } + } + } +} diff --git a/pkg/dagger.io/dagger/image.cue b/pkg/dagger.io/dagger/image.cue index 052d6f43..6068f63d 100644 --- a/pkg/dagger.io/dagger/image.cue +++ b/pkg/dagger.io/dagger/image.cue @@ -117,6 +117,32 @@ import ( config: #ImageConfig } +// Export an image as a tar archive +#Export: { + $dagger: task: _name: "Export" + + // Filesystem contents to export + input: #FS + + // Container image config + config: #ImageConfig + + // Name and optionally a tag in the 'name:tag' format + tag: string + + // Type of export + type: *"docker" | "oci" + + // Path to the exported file inside `output` + path: string | *"/image.tar" + + // Exported image ID + imageID: string + + // Root filesystem with exported file + output: #FS +} + // Change image config #Set: { // The source image config diff --git a/pkg/universe.dagger.io/docker/load.cue b/pkg/universe.dagger.io/docker/load.cue new file mode 100644 index 00000000..bb3df59e --- /dev/null +++ b/pkg/universe.dagger.io/docker/load.cue @@ -0,0 +1,112 @@ +package docker + +import ( + "dagger.io/dagger" +) + +// Load an image into a docker daemon +#Load: { + // Image to load + image: #Image + + // Name and optionally a tag in the 'name:tag' format + tag: #Ref + + // Exported image ID + imageID: _export.imageID + + // Root filesystem with exported file + result: _export.output + + _export: dagger.#Export & { + "tag": tag + input: image.rootfs + config: image.config + } + + #_cli & { + mounts: src: { + dest: "/src" + contents: _export.output + } + command: { + name: "load" + flags: "-i": "/src/image.tar" + } + } +} + +// FIXME: Move this into docker/client or +// create a better abstraction to reuse here. +#_cli: { + #_socketConn | #_sshConn | #_tcpConn + + _image: #Pull & { + source: "docker:20.10.13-alpine3.15" + } + + input: _image.output +} + +// Connect via local docker socket +#_socketConn: { + host: dagger.#Service + + #Run & { + mounts: docker: { + dest: "/var/run/docker.sock" + contents: host + } + } +} + +// Connect via HTTP/HTTPS +#_tcpConn: { + host: =~"^tcp://.+" + + #Run & { + env: DOCKER_HOST: host + + // Directory with certificates to verify ({ca,cert,key}.pem files). + // This enables HTTPS. + certs?: dagger.#FS + + if certs != _|_ { + mounts: "certs": { + dest: "/certs/client" + contents: certs + } + } + } +} + +// Connect via SSH +#_sshConn: { + host: =~"^ssh://.+" + + ssh: { + // Private SSH key + key?: dagger.#Secret + + // Known hosts file contents + knownHosts?: dagger.#Secret + } + + #Run & { + env: DOCKER_HOST: host + + if ssh.key != _|_ { + mounts: ssh_key: { + dest: "/root/.ssh/id_rsa" + contents: ssh.key + } + } + + if ssh.knownHosts != _|_ { + mounts: ssh_hosts: { + dest: "/root/.ssh/known_hosts" + contents: ssh.knownHosts + } + } + } +} diff --git a/pkg/universe.dagger.io/docker/test/load.cue b/pkg/universe.dagger.io/docker/test/load.cue new file mode 100644 index 00000000..84b0ec16 --- /dev/null +++ b/pkg/universe.dagger.io/docker/test/load.cue @@ -0,0 +1,57 @@ +package docker + +import ( + "dagger.io/dagger" + + "universe.dagger.io/alpine" + "universe.dagger.io/bash" + "universe.dagger.io/docker" +) + +dagger.#Plan & { + client: filesystem: "/var/run/docker.sock": read: contents: dagger.#Service + + actions: test: load: { + _cli: alpine.#Build & { + packages: { + bash: {} + "docker-cli": {} + } + } + + _image: docker.#Run & { + input: _cli.output + command: { + name: "touch" + args: ["/foo.bar"] + } + } + + load: docker.#Load & { + image: _image.output + host: client.filesystem."/var/run/docker.sock".read.contents + tag: "dagger:load" + } + + verify: bash.#Run & { + input: _cli.output + mounts: docker: { + contents: client.filesystem."/var/run/docker.sock".read.contents + dest: "/var/run/docker.sock" + } + env: { + IMAGE_NAME: load.tag + IMAGE_ID: load.imageID + // FIXME: without this forced dependency, load.command might not run + DEP: "\(load.success)" + } + script: contents: #""" + test "$(docker image inspect $IMAGE_NAME -f '{{.Id}}')" = "$IMAGE_ID" + docker run --rm $IMAGE_NAME stat /foo.bar + """# + } + } + + // FIXME: test remote connections with `docker:dind` + // image when we have long running tasks +} diff --git a/pkg/universe.dagger.io/docker/test/test.bats b/pkg/universe.dagger.io/docker/test/test.bats index bebf70cb..b126fcf6 100644 --- a/pkg/universe.dagger.io/docker/test/test.bats +++ b/pkg/universe.dagger.io/docker/test/test.bats @@ -9,4 +9,5 @@ setup() { dagger "do" -p ./dockerfile.cue test dagger "do" -p ./run.cue test dagger "do" -p ./image.cue test + dagger "do" -p ./load.cue test } diff --git a/plan/task/export.go b/plan/task/export.go new file mode 100644 index 00000000..92c7d9df --- /dev/null +++ b/plan/task/export.go @@ -0,0 +1,136 @@ +package task + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/moby/buildkit/exporter/containerimage/exptypes" + + "github.com/docker/distribution/reference" + bk "github.com/moby/buildkit/client" + "github.com/moby/buildkit/client/llb" + "github.com/rs/zerolog/log" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("Export", func() Task { return &exportTask{} }) +} + +type exportTask struct { +} + +func (t exportTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error { + dir, err := os.MkdirTemp("", "dagger-export-*") + if err != nil { + return err + } + + pctx.TempDirs.Add(dir, v.Path().String()) + pctx.LocalDirs.Add(dir) + + return nil +} + +func (t exportTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + lg := log.Ctx(ctx) + + dir := pctx.TempDirs.Get(v.Path().String()) + + var opts struct { + Tag string + Path string + Type string + } + + if err := v.Decode(&opts); err != nil { + return nil, err + } + + switch opts.Type { + case bk.ExporterDocker, bk.ExporterOCI: + default: + return nil, fmt.Errorf("unsupported export type %q", opts.Type) + } + + // Normalize tag + tag, err := reference.ParseNormalizedNamed(opts.Tag) + if err != nil { + return nil, fmt.Errorf("failed to parse ref %s: %w", opts.Tag, err) + } + tag = reference.TagNameOnly(tag) + + lg.Debug().Str("tag", tag.String()).Msg("normalized tag") + + // Get input state + input, err := pctx.FS.FromValue(v.Lookup("input")) + if err != nil { + return nil, err + } + st, err := input.State() + if err != nil { + return nil, err + } + + // Decode the image config + imageConfig := ImageConfig{} + if err := v.Lookup("config").Decode(&imageConfig); err != nil { + return nil, err + } + + img := NewImage(imageConfig, pctx.Platform.Get()) + + // Export image + resp, err := s.Export(ctx, st, &img, bk.ExportEntry{ + Type: opts.Type, + Attrs: map[string]string{ + "name": tag.String(), + }, + Output: func(a map[string]string) (io.WriteCloser, error) { + file := filepath.Join(dir, opts.Path) + return os.Create(file) + }, + }, pctx.Platform.Get()) + + if err != nil { + return nil, err + } + + // Save the image id + imageID, ok := resp.ExporterResponse[exptypes.ExporterImageConfigDigestKey] + if !ok { + return nil, fmt.Errorf("image export for %q did not return an image id", tag.String()) + } + + // FIXME: Remove the `Copy` and use `Local` directly. + // + // Copy'ing is a costly operation which should be unnecessary. + // However, using llb.Local directly breaks caching sometimes for unknown reasons. + outputState := llb.Scratch().File( + llb.Copy( + llb.Local( + dir, + withCustomName(v, "Export %s", opts.Path), + ), + "/", + "/", + ), + withCustomName(v, "Local %s [copy]", opts.Path), + ) + + result, err := s.Solve(ctx, outputState, pctx.Platform.Get()) + if err != nil { + return nil, err + } + + fs := pctx.FS.New(result) + return compiler.NewValue().FillFields(map[string]interface{}{ + "output": fs.MarshalCUE(), + "imageID": imageID, + }) +} diff --git a/plan/task/imageconfig.go b/plan/task/image.go similarity index 94% rename from plan/task/imageconfig.go rename to plan/task/image.go index a953f299..f18a7122 100644 --- a/plan/task/imageconfig.go +++ b/plan/task/image.go @@ -5,6 +5,7 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb" "github.com/moby/buildkit/frontend/dockerfile/shell" + specs "github.com/opencontainers/image-spec/specs-go/v1" ) // ImageConfig defines the execution parameters which should be used as a base when running a container using an image. @@ -152,3 +153,14 @@ func ConvertHealthConfig(spec *dockerfile2llb.HealthConfig) *HealthConfig { return &cfg } + +func NewImage(config ImageConfig, platform specs.Platform) dockerfile2llb.Image { + return dockerfile2llb.Image{ + Config: config.ToSpec(), + Image: specs.Image{ + Architecture: platform.Architecture, + OS: platform.OS, + }, + Variant: platform.Variant, + } +} diff --git a/plan/task/push.go b/plan/task/push.go index 4a7c4d04..1adf5546 100644 --- a/plan/task/push.go +++ b/plan/task/push.go @@ -6,7 +6,6 @@ import ( "github.com/docker/distribution/reference" bk "github.com/moby/buildkit/client" - "github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb" "github.com/rs/zerolog/log" "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/plancontext" @@ -67,15 +66,11 @@ func (c *pushTask) Run(ctx context.Context, pctx *plancontext.Context, s solver. return nil, err } - // Add platform to image configuration - exportImageConfig := &dockerfile2llb.Image{Config: imageConfig.ToSpec()} - exportImageConfig.OS = pctx.Platform.Get().OS - exportImageConfig.Architecture = pctx.Platform.Get().Architecture - exportImageConfig.Variant = pctx.Platform.Get().Variant + img := NewImage(imageConfig, pctx.Platform.Get()) // Export image lg.Debug().Str("dest", dest.String()).Msg("export image") - resp, err := s.Export(ctx, st, exportImageConfig, bk.ExportEntry{ + resp, err := s.Export(ctx, st, &img, bk.ExportEntry{ Type: bk.ExporterImage, Attrs: map[string]string{ "name": dest.String(), diff --git a/plancontext/context.go b/plancontext/context.go index 8b4fc548..58f6cb79 100644 --- a/plancontext/context.go +++ b/plancontext/context.go @@ -10,6 +10,7 @@ type Context struct { Platform *platformContext FS *fsContext LocalDirs *localDirContext + TempDirs *tempDirContext Secrets *secretContext Services *serviceContext } @@ -25,6 +26,9 @@ func New() *Context { LocalDirs: &localDirContext{ store: []string{}, }, + TempDirs: &tempDirContext{ + store: make(map[string]string), + }, Secrets: &secretContext{ store: make(map[string]*Secret), }, diff --git a/plancontext/tempdir.go b/plancontext/tempdir.go new file mode 100644 index 00000000..c9336afa --- /dev/null +++ b/plancontext/tempdir.go @@ -0,0 +1,34 @@ +package plancontext + +import ( + "os" + "sync" +) + +type tempDirContext struct { + l sync.RWMutex + store map[string]string +} + +func (c *tempDirContext) Add(dir, key string) { + c.l.Lock() + defer c.l.Unlock() + + c.store[key] = dir +} + +func (c *tempDirContext) Get(key string) string { + c.l.RLock() + defer c.l.RUnlock() + + return c.store[key] +} + +func (c *tempDirContext) Clean() { + c.l.RLock() + defer c.l.RUnlock() + + for _, s := range c.store { + defer os.RemoveAll(s) + } +} diff --git a/tests/tasks.bats b/tests/tasks.bats index a93e1af2..a57342a6 100644 --- a/tests/tasks.bats +++ b/tests/tasks.bats @@ -137,3 +137,6 @@ setup() { "$DAGGER" "do" -p ./tasks/diff/diff.cue test } +@test "task: #Export" { + "$DAGGER" "do" -p ./tasks/export/export.cue test +} diff --git a/tests/tasks/export/export.cue b/tests/tasks/export/export.cue new file mode 100644 index 00000000..d57276aa --- /dev/null +++ b/tests/tasks/export/export.cue @@ -0,0 +1,28 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + actions: test: { + image: dagger.#Pull & { + source: "alpine:3.15" + } + + export: dagger.#Export & { + input: image.output + config: image.config + tag: "example" + } + + verify: dagger.#Exec & { + input: image.output + mounts: exported: { + contents: export.output + dest: "/src" + } + args: ["tar", "tf", "/src/image.tar", "manifest.json"] + } + } +}