Merge pull request #1842 from helderco/docker-load

Add export and load for dagger images
This commit is contained in:
Andrea Luzzardi 2022-03-25 12:22:40 -07:00 committed by GitHub
commit 4f440bc67a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 488 additions and 7 deletions

View File

@ -73,6 +73,8 @@ var doCmd = &cobra.Command{
<-doneCh
p.Context().TempDirs.Clean()
if err != nil {
lg.Fatal().Err(err).Msg("failed to execute plan")
}

View File

@ -0,0 +1,20 @@
---
slug: /1216/engine-load
displayed_sidebar: europa
---
# Loading a dagger image into a docker daemon
Using `docker.#Load`, you can save a dagger image (`docker.#Image`) into a local or remote engine.
It can be useful to debug or test a build locally before pushing.
## Local daemon
```cue file=./plans/local.cue
```
## Remote daemon, via SSH
```cue file=./plans/ssh.cue
```

View File

@ -0,0 +1,22 @@
package main
import (
"dagger.io/dagger"
"universe.dagger.io/docker"
)
dagger.#Plan & {
client: filesystem: "/var/run/docker.sock": read: contents: dagger.#Service
actions: {
build: docker.#Build & {
...
}
load: docker.#Load & {
image: build.output
host: client.filesystem."/var/run/docker.sock".read.contents
tag: "myimage"
}
}
}

29
docs/drafts/plans/ssh.cue Normal file
View File

@ -0,0 +1,29 @@
package main
import (
"dagger.io/dagger"
"universe.dagger.io/docker"
)
dagger.#Plan & {
client: filesystem: {
"/home/user/.ssh/id_rsa": read: contents: dagger.#Secret
"/home/user/.ssh/known_hosts": read: contents: dagger.#Secret
}
actions: {
build: docker.#Build & {
...
}
load: docker.#Load & {
image: build.output
tag: "myimage:v2"
host: "ssh://root@93.184.216.34"
ssh: {
key: client.filesystem."/home/user/.ssh/id_rsa".read.contents
knownHosts: client.filesystem."/home/user/.ssh/known_hosts".read.contents
}
}
}
}

View File

@ -117,6 +117,32 @@ import (
config: #ImageConfig
}
// Export an image as a tar archive
#Export: {
$dagger: task: _name: "Export"
// Filesystem contents to export
input: #FS
// Container image config
config: #ImageConfig
// Name and optionally a tag in the 'name:tag' format
tag: string
// Type of export
type: *"docker" | "oci"
// Path to the exported file inside `output`
path: string | *"/image.tar"
// Exported image ID
imageID: string
// Root filesystem with exported file
output: #FS
}
// Change image config
#Set: {
// The source image config

View File

@ -0,0 +1,112 @@
package docker
import (
"dagger.io/dagger"
)
// Load an image into a docker daemon
#Load: {
// Image to load
image: #Image
// Name and optionally a tag in the 'name:tag' format
tag: #Ref
// Exported image ID
imageID: _export.imageID
// Root filesystem with exported file
result: _export.output
_export: dagger.#Export & {
"tag": tag
input: image.rootfs
config: image.config
}
#_cli & {
mounts: src: {
dest: "/src"
contents: _export.output
}
command: {
name: "load"
flags: "-i": "/src/image.tar"
}
}
}
// FIXME: Move this into docker/client or
// create a better abstraction to reuse here.
#_cli: {
#_socketConn | #_sshConn | #_tcpConn
_image: #Pull & {
source: "docker:20.10.13-alpine3.15"
}
input: _image.output
}
// Connect via local docker socket
#_socketConn: {
host: dagger.#Service
#Run & {
mounts: docker: {
dest: "/var/run/docker.sock"
contents: host
}
}
}
// Connect via HTTP/HTTPS
#_tcpConn: {
host: =~"^tcp://.+"
#Run & {
env: DOCKER_HOST: host
// Directory with certificates to verify ({ca,cert,key}.pem files).
// This enables HTTPS.
certs?: dagger.#FS
if certs != _|_ {
mounts: "certs": {
dest: "/certs/client"
contents: certs
}
}
}
}
// Connect via SSH
#_sshConn: {
host: =~"^ssh://.+"
ssh: {
// Private SSH key
key?: dagger.#Secret
// Known hosts file contents
knownHosts?: dagger.#Secret
}
#Run & {
env: DOCKER_HOST: host
if ssh.key != _|_ {
mounts: ssh_key: {
dest: "/root/.ssh/id_rsa"
contents: ssh.key
}
}
if ssh.knownHosts != _|_ {
mounts: ssh_hosts: {
dest: "/root/.ssh/known_hosts"
contents: ssh.knownHosts
}
}
}
}

View File

@ -0,0 +1,57 @@
package docker
import (
"dagger.io/dagger"
"universe.dagger.io/alpine"
"universe.dagger.io/bash"
"universe.dagger.io/docker"
)
dagger.#Plan & {
client: filesystem: "/var/run/docker.sock": read: contents: dagger.#Service
actions: test: load: {
_cli: alpine.#Build & {
packages: {
bash: {}
"docker-cli": {}
}
}
_image: docker.#Run & {
input: _cli.output
command: {
name: "touch"
args: ["/foo.bar"]
}
}
load: docker.#Load & {
image: _image.output
host: client.filesystem."/var/run/docker.sock".read.contents
tag: "dagger:load"
}
verify: bash.#Run & {
input: _cli.output
mounts: docker: {
contents: client.filesystem."/var/run/docker.sock".read.contents
dest: "/var/run/docker.sock"
}
env: {
IMAGE_NAME: load.tag
IMAGE_ID: load.imageID
// FIXME: without this forced dependency, load.command might not run
DEP: "\(load.success)"
}
script: contents: #"""
test "$(docker image inspect $IMAGE_NAME -f '{{.Id}}')" = "$IMAGE_ID"
docker run --rm $IMAGE_NAME stat /foo.bar
"""#
}
}
// FIXME: test remote connections with `docker:dind`
// image when we have long running tasks
}

View File

@ -9,4 +9,5 @@ setup() {
dagger "do" -p ./dockerfile.cue test
dagger "do" -p ./run.cue test
dagger "do" -p ./image.cue test
dagger "do" -p ./load.cue test
}

136
plan/task/export.go Normal file
View File

@ -0,0 +1,136 @@
package task
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
"github.com/docker/distribution/reference"
bk "github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/rs/zerolog/log"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/solver"
)
func init() {
Register("Export", func() Task { return &exportTask{} })
}
type exportTask struct {
}
func (t exportTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error {
dir, err := os.MkdirTemp("", "dagger-export-*")
if err != nil {
return err
}
pctx.TempDirs.Add(dir, v.Path().String())
pctx.LocalDirs.Add(dir)
return nil
}
func (t exportTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
dir := pctx.TempDirs.Get(v.Path().String())
var opts struct {
Tag string
Path string
Type string
}
if err := v.Decode(&opts); err != nil {
return nil, err
}
switch opts.Type {
case bk.ExporterDocker, bk.ExporterOCI:
default:
return nil, fmt.Errorf("unsupported export type %q", opts.Type)
}
// Normalize tag
tag, err := reference.ParseNormalizedNamed(opts.Tag)
if err != nil {
return nil, fmt.Errorf("failed to parse ref %s: %w", opts.Tag, err)
}
tag = reference.TagNameOnly(tag)
lg.Debug().Str("tag", tag.String()).Msg("normalized tag")
// Get input state
input, err := pctx.FS.FromValue(v.Lookup("input"))
if err != nil {
return nil, err
}
st, err := input.State()
if err != nil {
return nil, err
}
// Decode the image config
imageConfig := ImageConfig{}
if err := v.Lookup("config").Decode(&imageConfig); err != nil {
return nil, err
}
img := NewImage(imageConfig, pctx.Platform.Get())
// Export image
resp, err := s.Export(ctx, st, &img, bk.ExportEntry{
Type: opts.Type,
Attrs: map[string]string{
"name": tag.String(),
},
Output: func(a map[string]string) (io.WriteCloser, error) {
file := filepath.Join(dir, opts.Path)
return os.Create(file)
},
}, pctx.Platform.Get())
if err != nil {
return nil, err
}
// Save the image id
imageID, ok := resp.ExporterResponse[exptypes.ExporterImageConfigDigestKey]
if !ok {
return nil, fmt.Errorf("image export for %q did not return an image id", tag.String())
}
// FIXME: Remove the `Copy` and use `Local` directly.
//
// Copy'ing is a costly operation which should be unnecessary.
// However, using llb.Local directly breaks caching sometimes for unknown reasons.
outputState := llb.Scratch().File(
llb.Copy(
llb.Local(
dir,
withCustomName(v, "Export %s", opts.Path),
),
"/",
"/",
),
withCustomName(v, "Local %s [copy]", opts.Path),
)
result, err := s.Solve(ctx, outputState, pctx.Platform.Get())
if err != nil {
return nil, err
}
fs := pctx.FS.New(result)
return compiler.NewValue().FillFields(map[string]interface{}{
"output": fs.MarshalCUE(),
"imageID": imageID,
})
}

View File

@ -5,6 +5,7 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb"
"github.com/moby/buildkit/frontend/dockerfile/shell"
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
// ImageConfig defines the execution parameters which should be used as a base when running a container using an image.
@ -152,3 +153,14 @@ func ConvertHealthConfig(spec *dockerfile2llb.HealthConfig) *HealthConfig {
return &cfg
}
func NewImage(config ImageConfig, platform specs.Platform) dockerfile2llb.Image {
return dockerfile2llb.Image{
Config: config.ToSpec(),
Image: specs.Image{
Architecture: platform.Architecture,
OS: platform.OS,
},
Variant: platform.Variant,
}
}

View File

@ -6,7 +6,6 @@ import (
"github.com/docker/distribution/reference"
bk "github.com/moby/buildkit/client"
"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb"
"github.com/rs/zerolog/log"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/plancontext"
@ -67,15 +66,11 @@ func (c *pushTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.
return nil, err
}
// Add platform to image configuration
exportImageConfig := &dockerfile2llb.Image{Config: imageConfig.ToSpec()}
exportImageConfig.OS = pctx.Platform.Get().OS
exportImageConfig.Architecture = pctx.Platform.Get().Architecture
exportImageConfig.Variant = pctx.Platform.Get().Variant
img := NewImage(imageConfig, pctx.Platform.Get())
// Export image
lg.Debug().Str("dest", dest.String()).Msg("export image")
resp, err := s.Export(ctx, st, exportImageConfig, bk.ExportEntry{
resp, err := s.Export(ctx, st, &img, bk.ExportEntry{
Type: bk.ExporterImage,
Attrs: map[string]string{
"name": dest.String(),

View File

@ -10,6 +10,7 @@ type Context struct {
Platform *platformContext
FS *fsContext
LocalDirs *localDirContext
TempDirs *tempDirContext
Secrets *secretContext
Services *serviceContext
}
@ -25,6 +26,9 @@ func New() *Context {
LocalDirs: &localDirContext{
store: []string{},
},
TempDirs: &tempDirContext{
store: make(map[string]string),
},
Secrets: &secretContext{
store: make(map[string]*Secret),
},

34
plancontext/tempdir.go Normal file
View File

@ -0,0 +1,34 @@
package plancontext
import (
"os"
"sync"
)
type tempDirContext struct {
l sync.RWMutex
store map[string]string
}
func (c *tempDirContext) Add(dir, key string) {
c.l.Lock()
defer c.l.Unlock()
c.store[key] = dir
}
func (c *tempDirContext) Get(key string) string {
c.l.RLock()
defer c.l.RUnlock()
return c.store[key]
}
func (c *tempDirContext) Clean() {
c.l.RLock()
defer c.l.RUnlock()
for _, s := range c.store {
defer os.RemoveAll(s)
}
}

View File

@ -137,3 +137,6 @@ setup() {
"$DAGGER" "do" -p ./tasks/diff/diff.cue test
}
@test "task: #Export" {
"$DAGGER" "do" -p ./tasks/export/export.cue test
}

View File

@ -0,0 +1,28 @@
package main
import (
"dagger.io/dagger"
)
dagger.#Plan & {
actions: test: {
image: dagger.#Pull & {
source: "alpine:3.15"
}
export: dagger.#Export & {
input: image.output
config: image.config
tag: "example"
}
verify: dagger.#Exec & {
input: image.output
mounts: exported: {
contents: export.output
dest: "/src"
}
args: ["tar", "tf", "/src/image.tar", "manifest.json"]
}
}
}