Merge pull request #1842 from helderco/docker-load

Add export and load for dagger images

cmd/dagger/cmd/do.go

@@ -73,6 +73,8 @@ var doCmd = &cobra.Command{
 
 		<-doneCh
 
+		p.Context().TempDirs.Clean()
+
 		if err != nil {
 			lg.Fatal().Err(err).Msg("failed to execute plan")
 		}

docs/drafts/1216-engine-load.md

@@ -0,0 +1,20 @@
+---
+slug: /1216/engine-load
+displayed_sidebar: europa
+---
+
+# Loading a dagger image into a docker daemon
+
+Using `docker.#Load`, you can save a dagger image (`docker.#Image`) into a local or remote engine.
+
+It can be useful to debug or test a build locally before pushing.
+
+## Local daemon
+
+```cue file=./plans/local.cue
+```
+
+## Remote daemon, via SSH
+
+```cue file=./plans/ssh.cue
+```

docs/drafts/plans/local.cue

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"dagger.io/dagger"
+	"universe.dagger.io/docker"
+)
+
+dagger.#Plan & {
+	client: filesystem: "/var/run/docker.sock": read: contents: dagger.#Service
+
+	actions: {
+		build: docker.#Build & {
+			...
+		}
+
+		load: docker.#Load & {
+			image: build.output
+			host:  client.filesystem."/var/run/docker.sock".read.contents
+			tag:   "myimage"
+		}
+	}
+}

docs/drafts/plans/ssh.cue

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"dagger.io/dagger"
+	"universe.dagger.io/docker"
+)
+
+dagger.#Plan & {
+	client: filesystem: {
+		"/home/user/.ssh/id_rsa": read: contents:      dagger.#Secret
+		"/home/user/.ssh/known_hosts": read: contents: dagger.#Secret
+	}
+
+	actions: {
+		build: docker.#Build & {
+			...
+		}
+
+		load: docker.#Load & {
+			image: build.output
+			tag:   "myimage:v2"
+			host:  "ssh://root@93.184.216.34"
+			ssh: {
+				key:        client.filesystem."/home/user/.ssh/id_rsa".read.contents
+				knownHosts: client.filesystem."/home/user/.ssh/known_hosts".read.contents
+			}
+		}
+	}
+}

pkg/dagger.io/dagger/image.cue

@@ -117,6 +117,32 @@ import (
 	config: #ImageConfig
 }
 
+// Export an image as a tar archive
+#Export: {
+	$dagger: task: _name: "Export"
+
+	// Filesystem contents to export
+	input: #FS
+
+	// Container image config
+	config: #ImageConfig
+
+	// Name and optionally a tag in the 'name:tag' format
+	tag: string
+
+	// Type of export
+	type: *"docker" | "oci"
+
+	// Path to the exported file inside `output`
+	path: string | *"/image.tar"
+
+	// Exported image ID
+	imageID: string
+
+	// Root filesystem with exported file
+	output: #FS
+}
+
 // Change image config
 #Set: {
 	// The source image config

pkg/universe.dagger.io/docker/load.cue

@@ -0,0 +1,112 @@
+package docker
+
+import (
+	"dagger.io/dagger"
+)
+
+// Load an image into a docker daemon
+#Load: {
+	// Image to load
+	image: #Image
+
+	// Name and optionally a tag in the 'name:tag' format
+	tag: #Ref
+
+	// Exported image ID
+	imageID: _export.imageID
+
+	// Root filesystem with exported file
+	result: _export.output
+
+	_export: dagger.#Export & {
+		"tag":  tag
+		input:  image.rootfs
+		config: image.config
+	}
+
+	#_cli & {
+		mounts: src: {
+			dest:     "/src"
+			contents: _export.output
+		}
+		command: {
+			name: "load"
+			flags: "-i": "/src/image.tar"
+		}
+	}
+}
+
+// FIXME: Move this into docker/client or
+// create a better abstraction to reuse here.
+#_cli: {
+	#_socketConn | #_sshConn | #_tcpConn
+
+	_image: #Pull & {
+		source: "docker:20.10.13-alpine3.15"
+	}
+
+	input: _image.output
+}
+
+// Connect via local docker socket
+#_socketConn: {
+	host: dagger.#Service
+
+	#Run & {
+		mounts: docker: {
+			dest:     "/var/run/docker.sock"
+			contents: host
+		}
+	}
+}
+
+// Connect via HTTP/HTTPS
+#_tcpConn: {
+	host: =~"^tcp://.+"
+
+	#Run & {
+		env: DOCKER_HOST: host
+
+		// Directory with certificates to verify ({ca,cert,key}.pem files).
+		// This enables HTTPS.
+		certs?: dagger.#FS
+
+		if certs != _|_ {
+			mounts: "certs": {
+				dest:     "/certs/client"
+				contents: certs
+			}
+		}
+	}
+}
+
+// Connect via SSH
+#_sshConn: {
+	host: =~"^ssh://.+"
+
+	ssh: {
+		// Private SSH key
+		key?: dagger.#Secret
+
+		// Known hosts file contents
+		knownHosts?: dagger.#Secret
+	}
+
+	#Run & {
+		env: DOCKER_HOST: host
+
+		if ssh.key != _|_ {
+			mounts: ssh_key: {
+				dest:     "/root/.ssh/id_rsa"
+				contents: ssh.key
+			}
+		}
+
+		if ssh.knownHosts != _|_ {
+			mounts: ssh_hosts: {
+				dest:     "/root/.ssh/known_hosts"
+				contents: ssh.knownHosts
+			}
+		}
+	}
+}

pkg/universe.dagger.io/docker/test/load.cue

@@ -0,0 +1,57 @@
+package docker
+
+import (
+	"dagger.io/dagger"
+
+	"universe.dagger.io/alpine"
+	"universe.dagger.io/bash"
+	"universe.dagger.io/docker"
+)
+
+dagger.#Plan & {
+	client: filesystem: "/var/run/docker.sock": read: contents: dagger.#Service
+
+	actions: test: load: {
+		_cli: alpine.#Build & {
+			packages: {
+				bash: {}
+				"docker-cli": {}
+			}
+		}
+
+		_image: docker.#Run & {
+			input: _cli.output
+			command: {
+				name: "touch"
+				args: ["/foo.bar"]
+			}
+		}
+
+		load: docker.#Load & {
+			image: _image.output
+			host:  client.filesystem."/var/run/docker.sock".read.contents
+			tag:   "dagger:load"
+		}
+
+		verify: bash.#Run & {
+			input: _cli.output
+			mounts: docker: {
+				contents: client.filesystem."/var/run/docker.sock".read.contents
+				dest:     "/var/run/docker.sock"
+			}
+			env: {
+				IMAGE_NAME: load.tag
+				IMAGE_ID:   load.imageID
+				// FIXME: without this forced dependency, load.command might not run
+				DEP: "\(load.success)"
+			}
+			script: contents: #"""
+				test "$(docker image inspect $IMAGE_NAME -f '{{.Id}}')" = "$IMAGE_ID"
+				docker run --rm $IMAGE_NAME stat /foo.bar
+				"""#
+		}
+	}
+
+	// FIXME: test remote connections with `docker:dind`
+	// image when we have long running tasks
+}

pkg/universe.dagger.io/docker/test/test.bats

@@ -9,4 +9,5 @@ setup() {
     dagger "do" -p ./dockerfile.cue test
     dagger "do" -p ./run.cue test
     dagger "do" -p ./image.cue test
+    dagger "do" -p ./load.cue test
 }

plan/task/export.go

@@ -0,0 +1,136 @@
+package task
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/moby/buildkit/exporter/containerimage/exptypes"
+
+	"github.com/docker/distribution/reference"
+	bk "github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/client/llb"
+	"github.com/rs/zerolog/log"
+	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/plancontext"
+	"go.dagger.io/dagger/solver"
+)
+
+func init() {
+	Register("Export", func() Task { return &exportTask{} })
+}
+
+type exportTask struct {
+}
+
+func (t exportTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error {
+	dir, err := os.MkdirTemp("", "dagger-export-*")
+	if err != nil {
+		return err
+	}
+
+	pctx.TempDirs.Add(dir, v.Path().String())
+	pctx.LocalDirs.Add(dir)
+
+	return nil
+}
+
+func (t exportTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	lg := log.Ctx(ctx)
+
+	dir := pctx.TempDirs.Get(v.Path().String())
+
+	var opts struct {
+		Tag  string
+		Path string
+		Type string
+	}
+
+	if err := v.Decode(&opts); err != nil {
+		return nil, err
+	}
+
+	switch opts.Type {
+	case bk.ExporterDocker, bk.ExporterOCI:
+	default:
+		return nil, fmt.Errorf("unsupported export type %q", opts.Type)
+	}
+
+	// Normalize tag
+	tag, err := reference.ParseNormalizedNamed(opts.Tag)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse ref %s: %w", opts.Tag, err)
+	}
+	tag = reference.TagNameOnly(tag)
+
+	lg.Debug().Str("tag", tag.String()).Msg("normalized tag")
+
+	// Get input state
+	input, err := pctx.FS.FromValue(v.Lookup("input"))
+	if err != nil {
+		return nil, err
+	}
+	st, err := input.State()
+	if err != nil {
+		return nil, err
+	}
+
+	// Decode the image config
+	imageConfig := ImageConfig{}
+	if err := v.Lookup("config").Decode(&imageConfig); err != nil {
+		return nil, err
+	}
+
+	img := NewImage(imageConfig, pctx.Platform.Get())
+
+	// Export image
+	resp, err := s.Export(ctx, st, &img, bk.ExportEntry{
+		Type: opts.Type,
+		Attrs: map[string]string{
+			"name": tag.String(),
+		},
+		Output: func(a map[string]string) (io.WriteCloser, error) {
+			file := filepath.Join(dir, opts.Path)
+			return os.Create(file)
+		},
+	}, pctx.Platform.Get())
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Save the image id
+	imageID, ok := resp.ExporterResponse[exptypes.ExporterImageConfigDigestKey]
+	if !ok {
+		return nil, fmt.Errorf("image export for %q did not return an image id", tag.String())
+	}
+
+	// FIXME: Remove the `Copy` and use `Local` directly.
+	//
+	// Copy'ing is a costly operation which should be unnecessary.
+	// However, using llb.Local directly breaks caching sometimes for unknown reasons.
+	outputState := llb.Scratch().File(
+		llb.Copy(
+			llb.Local(
+				dir,
+				withCustomName(v, "Export %s", opts.Path),
+			),
+			"/",
+			"/",
+		),
+		withCustomName(v, "Local %s [copy]", opts.Path),
+	)
+
+	result, err := s.Solve(ctx, outputState, pctx.Platform.Get())
+	if err != nil {
+		return nil, err
+	}
+
+	fs := pctx.FS.New(result)
+	return compiler.NewValue().FillFields(map[string]interface{}{
+		"output":  fs.MarshalCUE(),
+		"imageID": imageID,
+	})
+}

plan/task/image.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb"
 	"github.com/moby/buildkit/frontend/dockerfile/shell"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ImageConfig defines the execution parameters which should be used as a base when running a container using an image.
@@ -152,3 +153,14 @@ func ConvertHealthConfig(spec *dockerfile2llb.HealthConfig) *HealthConfig {
 
 	return &cfg
 }
+
+func NewImage(config ImageConfig, platform specs.Platform) dockerfile2llb.Image {
+	return dockerfile2llb.Image{
+		Config: config.ToSpec(),
+		Image: specs.Image{
+			Architecture: platform.Architecture,
+			OS:           platform.OS,
+		},
+		Variant: platform.Variant,
+	}
+}

plan/task/push.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/docker/distribution/reference"
 	bk "github.com/moby/buildkit/client"
-	"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb"
 	"github.com/rs/zerolog/log"
 	"go.dagger.io/dagger/compiler"
 	"go.dagger.io/dagger/plancontext"
@@ -67,15 +66,11 @@ func (c *pushTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.
 		return nil, err
 	}
 
-	// Add platform to image configuration
+	img := NewImage(imageConfig, pctx.Platform.Get())
-	exportImageConfig := &dockerfile2llb.Image{Config: imageConfig.ToSpec()}
-	exportImageConfig.OS = pctx.Platform.Get().OS
-	exportImageConfig.Architecture = pctx.Platform.Get().Architecture
-	exportImageConfig.Variant = pctx.Platform.Get().Variant
 
 	// Export image
 	lg.Debug().Str("dest", dest.String()).Msg("export image")
-	resp, err := s.Export(ctx, st, exportImageConfig, bk.ExportEntry{
+	resp, err := s.Export(ctx, st, &img, bk.ExportEntry{
 		Type: bk.ExporterImage,
 		Attrs: map[string]string{
 			"name": dest.String(),

plancontext/context.go

@@ -10,6 +10,7 @@ type Context struct {
 	Platform  *platformContext
 	FS        *fsContext
 	LocalDirs *localDirContext
+	TempDirs  *tempDirContext
 	Secrets   *secretContext
 	Services  *serviceContext
 }
@@ -25,6 +26,9 @@ func New() *Context {
 		LocalDirs: &localDirContext{
 			store: []string{},
 		},
+		TempDirs: &tempDirContext{
+			store: make(map[string]string),
+		},
 		Secrets: &secretContext{
 			store: make(map[string]*Secret),
 		},

plancontext/tempdir.go

@@ -0,0 +1,34 @@
+package plancontext
+
+import (
+	"os"
+	"sync"
+)
+
+type tempDirContext struct {
+	l     sync.RWMutex
+	store map[string]string
+}
+
+func (c *tempDirContext) Add(dir, key string) {
+	c.l.Lock()
+	defer c.l.Unlock()
+
+	c.store[key] = dir
+}
+
+func (c *tempDirContext) Get(key string) string {
+	c.l.RLock()
+	defer c.l.RUnlock()
+
+	return c.store[key]
+}
+
+func (c *tempDirContext) Clean() {
+	c.l.RLock()
+	defer c.l.RUnlock()
+
+	for _, s := range c.store {
+		defer os.RemoveAll(s)
+	}
+}

tests/tasks.bats

@@ -137,3 +137,6 @@ setup() {
     "$DAGGER" "do" -p ./tasks/diff/diff.cue test
 }
 
+@test "task: #Export" {
+    "$DAGGER" "do" -p ./tasks/export/export.cue test
+}

tests/tasks/export/export.cue

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"dagger.io/dagger"
+)
+
+dagger.#Plan & {
+	actions: test: {
+		image: dagger.#Pull & {
+			source: "alpine:3.15"
+		}
+
+		export: dagger.#Export & {
+			input:  image.output
+			config: image.config
+			tag:    "example"
+		}
+
+		verify: dagger.#Exec & {
+			input: image.output
+			mounts: exported: {
+				contents: export.output
+				dest:     "/src"
+			}
+			args: ["tar", "tf", "/src/image.tar", "manifest.json"]
+		}
+	}
+}