Merge pull request #134 from dagger/dockerbuild

docker build support

Dockerfile

@@ -1,4 +1,4 @@
-# syntax = docker/dockerfile-upstream:experimental@sha256:398a0a10f19875add7fe359a37f2f971c46746b064faf876776ae632a3472c37
+# syntax = docker/dockerfile:1.2
 
 FROM golang:1.16-alpine AS build
 WORKDIR /src

dagger/fs.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/moby/buildkit/client/llb"
 	bkgw "github.com/moby/buildkit/frontend/gateway/client"
+	bkpb "github.com/moby/buildkit/solver/pb"
 	fstypes "github.com/tonistiigi/fsutil/types"
 
 	"dagger.io/go/dagger/compiler"
@@ -162,6 +163,14 @@ func (fs FS) LLB() llb.State {
 	return fs.input
 }
 
+func (fs FS) Def(ctx context.Context) (*bkpb.Definition, error) {
+	def, err := fs.LLB().Marshal(ctx, llb.LinuxAmd64)
+	if err != nil {
+		return nil, err
+	}
+	return def.ToPB(), nil
+}
+
 func (fs FS) Ref(ctx context.Context) (bkgw.Reference, error) {
 	if err := (&fs).solve(ctx); err != nil {
 		return nil, err

dagger/pipeline.go

@@ -3,9 +3,14 @@ package dagger
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/moby/buildkit/client/llb"
+	dockerfilebuilder "github.com/moby/buildkit/frontend/dockerfile/builder"
+	bkgw "github.com/moby/buildkit/frontend/gateway/client"
+	bkpb "github.com/moby/buildkit/solver/pb"
 	"github.com/rs/zerolog/log"
 	"gopkg.in/yaml.v3"
 
@@ -157,6 +162,8 @@ func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value) error {
 		return p.Load(ctx, op)
 	case "subdir":
 		return p.Subdir(ctx, op)
+	case "docker-build":
+		return p.DockerBuild(ctx, op)
 	default:
 		return fmt.Errorf("invalid operation: %s", op.JSON())
 	}
@@ -431,6 +438,7 @@ func (p *Pipeline) Load(ctx context.Context, op *compiler.Value) error {
 	if err := from.Do(ctx, op.Get("from")); err != nil {
 		return err
 	}
+
 	p.fs = p.fs.Set(from.FS().LLB())
 	return nil
 }
@@ -457,3 +465,132 @@ func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value) error {
 	p.fs = p.fs.Set(llb.Git(remote, ref))
 	return nil
 }
+
+func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error {
+	var (
+		context    = op.Lookup("context")
+		dockerfile = op.Lookup("dockerfile")
+
+		contextDef    *bkpb.Definition
+		dockerfileDef *bkpb.Definition
+
+		err error
+	)
+
+	if !context.Exists() && !dockerfile.Exists() {
+		return errors.New("context or dockerfile required")
+	}
+
+	// docker build context. This can come from another component, so we need to
+	// compute it first.
+	if context.Exists() {
+		from := p.Tmp()
+		if err := from.Do(ctx, context); err != nil {
+			return err
+		}
+		contextDef, err = from.FS().Def(ctx)
+		if err != nil {
+			return err
+		}
+		dockerfileDef = contextDef
+	}
+
+	// Inlined dockerfile: need to be converted to LLB
+	if dockerfile.Exists() {
+		content, err := dockerfile.String()
+		if err != nil {
+			return err
+		}
+		dockerfileDef, err = p.s.Scratch().Set(
+			llb.Scratch().File(
+				llb.Mkfile("/Dockerfile", 0644, []byte(content)),
+			),
+		).Def(ctx)
+		if err != nil {
+			return err
+		}
+		if contextDef == nil {
+			contextDef = dockerfileDef
+		}
+	}
+
+	req := bkgw.SolveRequest{
+		Frontend:    "dockerfile.v0",
+		FrontendOpt: make(map[string]string),
+		FrontendInputs: map[string]*bkpb.Definition{
+			dockerfilebuilder.DefaultLocalNameContext:    contextDef,
+			dockerfilebuilder.DefaultLocalNameDockerfile: dockerfileDef,
+		},
+	}
+
+	if dockerfilePath := op.Lookup("dockerfilePath"); dockerfilePath.Exists() {
+		filename, err := dockerfilePath.String()
+		if err != nil {
+			return err
+		}
+		req.FrontendOpt["filename"] = filename
+	}
+
+	if buildArgs := op.Lookup("buildArg"); buildArgs.Exists() {
+		err := buildArgs.RangeStruct(func(key string, value *compiler.Value) error {
+			v, err := value.String()
+			if err != nil {
+				return err
+			}
+			req.FrontendOpt["build-arg:"+key] = v
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	if labels := op.Lookup("label"); labels.Exists() {
+		err := labels.RangeStruct(func(key string, value *compiler.Value) error {
+			s, err := value.String()
+			if err != nil {
+				return err
+			}
+			req.FrontendOpt["label:"+key] = s
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	if platforms := op.Lookup("platforms"); platforms.Exists() {
+		p := []string{}
+		list, err := platforms.List()
+		if err != nil {
+			return err
+		}
+
+		for _, platform := range list {
+			s, err := platform.String()
+			if err != nil {
+				return err
+			}
+			p = append(p, s)
+		}
+
+		if len(p) > 0 {
+			req.FrontendOpt["platform"] = strings.Join(p, ",")
+		}
+		if len(p) > 1 {
+			req.FrontendOpt["multi-platform"] = "true"
+		}
+	}
+
+	res, err := p.s.SolveRequest(ctx, req)
+	if err != nil {
+		return err
+	}
+	st, err := res.ToState()
+	if err != nil {
+		return err
+	}
+	p.fs = p.fs.Set(st)
+
+	return nil
+}

dagger/solver.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/moby/buildkit/client/llb"
 	bkgw "github.com/moby/buildkit/frontend/gateway/client"
-	"github.com/moby/buildkit/solver/pb"
+	bkpb "github.com/moby/buildkit/solver/pb"
 	"github.com/opencontainers/go-digest"
 	"github.com/rs/zerolog/log"
 )
@@ -35,6 +35,17 @@ func (s Solver) Scratch() FS {
 	return s.FS(llb.Scratch())
 }
 
+// Solve will block until the state is solved and returns a Reference.
+func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (bkgw.Reference, error) {
+	// call solve
+	res, err := s.c.Solve(ctx, req)
+	if err != nil {
+		return nil, bkCleanError(err)
+	}
+	// always use single reference (ignore multiple outputs & metadata)
+	return res.SingleRef()
+}
+
 // Solve will block until the state is solved and returns a Reference.
 func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) {
 	// marshal llb
@@ -55,7 +66,7 @@ func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error)
 		Msg("solving")
 
 	// call solve
-	res, err := s.c.Solve(ctx, bkgw.SolveRequest{
+	return s.SolveRequest(ctx, bkgw.SolveRequest{
 		Definition: def.ToPB(),
 
 		// makes Solve() to block until LLB graph is solved. otherwise it will
@@ -63,23 +74,18 @@ func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error)
 		// will be evaluated on export or if you access files on it.
 		Evaluate: true,
 	})
-	if err != nil {
-		return nil, bkCleanError(err)
-	}
-	// always use single reference (ignore multiple outputs & metadata)
-	return res.SingleRef()
 }
 
 type llbOp struct {
-	Op         pb.Op
+	Op         bkpb.Op
 	Digest     digest.Digest
-	OpMetadata pb.OpMetadata
+	OpMetadata bkpb.OpMetadata
 }
 
 func dumpLLB(def *llb.Definition) ([]byte, error) {
 	ops := make([]llbOp, 0, len(def.Def))
 	for _, dt := range def.Def {
-		var op pb.Op
+		var op bkpb.Op
 		if err := (&op).Unmarshal(dt); err != nil {
 			return nil, fmt.Errorf("failed to parse op: %w", err)
 		}

examples/dogfood/main.cue

@@ -7,6 +7,7 @@ import (
 
 repository: dagger.#Dir // Use `--input-dir repository=.` from the root directory of the project
 
+// Build `dagger` using Go
 build: go.#Build & {
 	source:   repository
 	packages: "./cmd/dagger"
@@ -18,7 +19,21 @@ test: go.#Test & {
 	packages: "./..."
 }
 
+// Run a command with the binary we just built
 help: #dagger: compute: [
 	dagger.#Load & {from: build},
 	dagger.#Exec & {args: ["dagger", "-h"]},
 ]
+
+// Build dagger using the (included) Dockerfile
+buildWithDocker: #dagger: compute: [
+	dagger.#DockerBuild & {
+		context: repository
+	},
+]
+
+// Run a command in the docker image we just built
+helpFromDocker: #dagger: compute: [
+	dagger.#Load & {from: buildWithDocker},
+	dagger.#Exec & {args: ["dagger", "-h"]},
+]

stdlib/dagger/dagger.cue

@@ -59,3 +59,15 @@ package dagger
 	src:  string | *"/"
 	dest: string | *"/"
 }
+
+#DockerBuild: {
+	do: "docker-build"
+	// We accept either a context, a Dockerfile or both together
+	context?:        _
+	dockerfilePath?: string // path to the Dockerfile (defaults to "Dockerfile")
+	dockerfile?:     string
+
+	platforms?: [...string]
+	buildArg?: [string]: string
+	label?: [string]:    string
+}

tests/dockerbuild/main.cue

@@ -0,0 +1,90 @@
+package test
+
+import "dagger.io/dagger"
+
+// Set to `--input-dir=./tests/dockerbuild/testdata`
+TestData: dagger.#Dir
+
+TestInlinedDockerfile: #dagger: compute: [
+	dagger.#DockerBuild & {
+		dockerfile: """
+			FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+			RUN echo hello world
+			"""
+	},
+]
+
+TestOpChaining: #dagger: compute: [
+	dagger.#DockerBuild & {
+		dockerfile: """
+			FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+			RUN echo foobar > /output
+			"""
+	},
+	dagger.#Exec & {
+		args: ["sh", "-c", "test $(cat /output) = foobar"]
+	}
+]
+
+TestBuildContext: #dagger: compute: [
+	dagger.#DockerBuild & {
+		context: TestData
+	},
+	dagger.#Exec & {
+		args: ["sh", "-c", "test $(cat /dir/foo) = foobar"]
+	}
+]
+
+TestBuildContextAndDockerfile: #dagger: compute: [
+	dagger.#DockerBuild & {
+		context: TestData
+		dockerfile: """
+			FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+			COPY foo /override
+			"""
+	},
+	dagger.#Exec & {
+		args: ["sh", "-c", "test $(cat /override) = foobar"]
+	}
+]
+
+TestDockerfilePath: #dagger: compute: [
+	dagger.#DockerBuild & {
+		context: TestData
+		dockerfilePath: "./dockerfilepath/Dockerfile.custom"
+	},
+	dagger.#Exec & {
+		args: ["sh", "-c", "test $(cat /test) = dockerfilePath"]
+	}
+]
+
+TestBuildArgs: #dagger: compute: [
+	dagger.#DockerBuild & {
+		dockerfile: """
+			FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+			ARG TEST=foo
+			RUN test "${TEST}" = "bar"
+			"""
+		buildArg: TEST: "bar"
+	}
+]
+
+// FIXME: this doesn't test anything beside not crashing
+TestBuildLabels: #dagger: compute: [
+	dagger.#DockerBuild & {
+		dockerfile: """
+			FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+			"""
+		label: FOO: "bar"
+	}
+]
+
+// FIXME: this doesn't test anything beside not crashing
+TestBuildPlatform: #dagger: compute: [
+	dagger.#DockerBuild & {
+		dockerfile: """
+			FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+			"""
+		platforms: ["linux/amd64"]
+	}
+]

tests/dockerbuild/testdata/Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+COPY . /dir
+RUN test $(cat /dir/foo) = foobar

tests/dockerbuild/testdata/dockerfilepath/Dockerfile.custom

@@ -0,0 +1,2 @@
+FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+RUN echo dockerfilePath > /test

tests/dockerbuild/testdata/foo

@@ -0,0 +1 @@
+foobar

tests/test.sh

@@ -229,6 +229,11 @@ test::subdir() {
       "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/subdir/simple
 }
 
+test::dockerbuild() {
+  test::one "Docker Build" --exit=0 \
+      "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute --input-dir TestData="$d"/dockerbuild/testdata "$d"/dockerbuild
+}
+
 test::all(){
   local dagger="$1"
 
@@ -244,6 +249,7 @@ test::all(){
   test::export "$dagger"
   test::input "$dagger"
   test::subdir "$dagger"
+  test::dockerbuild "$dagger"
 
   test::examples "$dagger"
 }