Merge pull request #1276 from aluzzardi/engine-build

engine.#Build support

plan/task/build.go

@@ -0,0 +1,227 @@
+package task
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	bkplatforms "github.com/containerd/containerd/platforms"
+	"github.com/moby/buildkit/client/llb"
+	"github.com/moby/buildkit/exporter/containerimage/exptypes"
+	dockerfilebuilder "github.com/moby/buildkit/frontend/dockerfile/builder"
+	"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb"
+	bkgw "github.com/moby/buildkit/frontend/gateway/client"
+	bkpb "github.com/moby/buildkit/solver/pb"
+	"github.com/rs/zerolog/log"
+
+	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/plancontext"
+	"go.dagger.io/dagger/solver"
+)
+
+func init() {
+	Register("Build", func() Task { return &buildTask{} })
+}
+
+type buildTask struct {
+}
+
+func (t *buildTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	frontend, err := v.Lookup("frontend").String()
+	if err != nil {
+		return nil, err
+	}
+
+	switch frontend {
+	case "dockerfile":
+		return t.dockerfile(ctx, pctx, s, v)
+	default:
+		return nil, fmt.Errorf("unsupported frontend %q", frontend)
+	}
+}
+
+func (t *buildTask) dockerfile(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	lg := log.Ctx(ctx)
+
+	// Read auth info
+	auth, err := decodeAuthValue(pctx, v.Lookup("auth"))
+	if err != nil {
+		return nil, err
+	}
+	for _, a := range auth {
+		s.AddCredentials(a.Target, a.Username, a.Secret.PlainText())
+		lg.Debug().Str("target", a.Target).Msg("add target credentials")
+	}
+
+	source, err := pctx.FS.FromValue(v.Lookup("source"))
+	if err != nil {
+		return nil, err
+	}
+
+	sourceSt, err := source.Result().ToState()
+	if err != nil {
+		return nil, err
+	}
+
+	// docker build context
+	contextDef, err := s.Marshal(ctx, sourceSt)
+	if err != nil {
+		return nil, err
+	}
+	// Dockerfile context, default to docker build context
+	dockerfileDef := contextDef
+
+	// Support inlined dockerfile
+	if dockerfile := v.Lookup("dockerfile.contents"); dockerfile.Exists() {
+		contents, err := dockerfile.String()
+		if err != nil {
+			return nil, err
+		}
+		dockerfileDef, err = s.Marshal(ctx,
+			llb.Scratch().File(
+				llb.Mkfile("/Dockerfile", 0644, []byte(contents)),
+			),
+		)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	opts, err := t.dockerBuildOpts(v, pctx)
+	if err != nil {
+		return nil, err
+	}
+	// Handle --no-cache
+	if s.NoCache() {
+		opts["no-cache"] = ""
+	}
+
+	req := bkgw.SolveRequest{
+		Frontend:    "dockerfile.v0",
+		FrontendOpt: opts,
+		FrontendInputs: map[string]*bkpb.Definition{
+			dockerfilebuilder.DefaultLocalNameContext:    contextDef,
+			dockerfilebuilder.DefaultLocalNameDockerfile: dockerfileDef,
+		},
+	}
+	res, err := s.SolveRequest(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	ref, err := res.SingleRef()
+	if err != nil {
+		return nil, err
+	}
+
+	// Image metadata
+	meta, ok := res.Metadata[exptypes.ExporterImageConfigKey]
+	if !ok {
+		return nil, errors.New("build returned no image config")
+	}
+	var image dockerfile2llb.Image
+	if err := json.Unmarshal(meta, &image); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal image config: %w", err)
+	}
+
+	return compiler.NewValue().FillFields(map[string]interface{}{
+		"output": pctx.FS.New(ref).MarshalCUE(),
+		"config": image.Config,
+	})
+}
+
+func (t *buildTask) dockerBuildOpts(v *compiler.Value, pctx *plancontext.Context) (map[string]string, error) {
+	opts := map[string]string{}
+
+	if dockerfilePath := v.Lookup("dockerfile.path"); dockerfilePath.Exists() {
+		filename, err := dockerfilePath.String()
+		if err != nil {
+			return nil, err
+		}
+		opts["filename"] = filename
+	}
+
+	if target := v.Lookup("target"); target.Exists() {
+		tgr, err := target.String()
+		if err != nil {
+			return nil, err
+		}
+		opts["target"] = tgr
+	}
+
+	if hosts := v.Lookup("hosts"); hosts.Exists() {
+		p := []string{}
+		fields, err := hosts.Fields()
+		if err != nil {
+			return nil, err
+		}
+		for _, host := range fields {
+			s, err := host.Value.String()
+			if err != nil {
+				return nil, err
+			}
+			p = append(p, host.Label()+"="+s)
+		}
+		if len(p) > 0 {
+			opts["add-hosts"] = strings.Join(p, ",")
+		}
+	}
+
+	if buildArgs := v.Lookup("buildArg"); buildArgs.Exists() {
+		fields, err := buildArgs.Fields()
+		if err != nil {
+			return nil, err
+		}
+		for _, buildArg := range fields {
+			s, err := buildArg.Value.String()
+			if err != nil {
+				return nil, err
+			}
+			opts["build-arg:"+buildArg.Label()] = s
+		}
+	}
+
+	if labels := v.Lookup("label"); labels.Exists() {
+		fields, err := labels.Fields()
+		if err != nil {
+			return nil, err
+		}
+		for _, label := range fields {
+			s, err := label.Value.String()
+			if err != nil {
+				return nil, err
+			}
+			opts["label:"+label.Label()] = s
+		}
+	}
+
+	if platforms := v.Lookup("platforms"); platforms.Exists() {
+		p := []string{}
+		list, err := platforms.List()
+		if err != nil {
+			return nil, err
+		}
+
+		for _, platform := range list {
+			s, err := platform.String()
+			if err != nil {
+				return nil, err
+			}
+			p = append(p, s)
+		}
+
+		if len(p) > 0 {
+			opts["platform"] = strings.Join(p, ",")
+		}
+		if len(p) > 1 {
+			opts["multi-platform"] = "true"
+		}
+	}
+	// Set platform to configured one if no one is defined
+	if opts["platform"] == "" {
+		opts["platform"] = bkplatforms.Format(pctx.Platform.Get())
+	}
+
+	return opts, nil
+}

stdlib/europa/dagger/engine/image.cue

@@ -71,7 +71,6 @@ package engine
 // Build a container image using buildkit
 // FIXME: rename to #Dockerfile to clarify scope
 #Build: {
-	@dagger(notimplemented)
 	$dagger: task: _name: "Build"
 
 	// Source directory to build
@@ -83,6 +82,20 @@ package engine
 		} | {
 			contents: string
 		}
+
+		// Authentication
+		auth: [...{
+			target:   string
+			username: string
+			secret:   string | #Secret
+		}]
+
+		// FIXME: options ported from op.#DockerBuild
+		platforms?: [...string]
+		target?: string
+		buildArg?: [string]: string
+		label?: [string]:    string
+		hosts?: [string]:    string
 	}
 
 	// Root filesystem produced by build

tests/tasks.bats

@@ -60,7 +60,6 @@ setup() {
     assert_failure
 }
 
-
 @test "task: #Mkdir" {
     # Make directory
     cd "$TESTDIR"/tasks/mkdir
@@ -75,3 +74,17 @@ setup() {
     run "$DAGGER" --europa up ./mkdir_failure_disable_parents.cue
     assert_failure
 }
+
+@test "task: #Build" {
+    cd "$TESTDIR"/tasks/build
+
+    "$DAGGER" --europa up ./dockerfile.cue
+    "$DAGGER" --europa up ./inlined_dockerfile.cue
+    "$DAGGER" --europa up ./dockerfile_path.cue
+    "$DAGGER" --europa up ./build_args.cue
+    "$DAGGER" --europa up ./image_config.cue
+    "$DAGGER" --europa up ./labels.cue
+    "$DAGGER" --europa up ./platform.cue
+
+    "$DAGGER" --europa up ./build_auth.cue
+}

tests/tasks/build/build_args.cue

@@ -0,0 +1,19 @@
+package testing
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	inputs: directories: testdata: path: "./testdata"
+
+	actions: build: engine.#Build & {
+		source: inputs.directories.testdata.contents
+		dockerfile: contents: """
+			FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+			ARG TEST=foo
+			RUN test "${TEST}" = "bar"
+			"""
+		buildArg: TEST: "bar"
+	}
+}

tests/tasks/build/build_auth.cue

@@ -0,0 +1,24 @@
+package testing
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	inputs: {
+		directories: testdata: path:     "./testdata"
+		secrets: dockerHubToken: envvar: "DOCKERHUB_TOKEN"
+	}
+
+	actions: build: engine.#Build & {
+		source: inputs.directories.testdata.contents
+		auth: [{
+			target:   "daggerio/ci-test:private-pull"
+			username: "daggertest"
+			secret:   inputs.secrets.dockerHubToken.contents
+		}]
+		dockerfile: contents: """
+			FROM daggerio/ci-test:private-pull@sha256:c74f1b1166784193ea6c8f9440263b9be6cae07dfe35e32a5df7a31358ac2060
+			"""
+	}
+}

tests/tasks/build/cue.mod/module.cue

@@ -0,0 +1 @@
+module: ""

tests/tasks/build/cue.mod/pkg/.gitignore

@@ -0,0 +1,3 @@
+# generated by dagger
+alpha.dagger.io
+dagger.lock

tests/tasks/build/dockerfile.cue

@@ -0,0 +1,20 @@
+package testing
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	inputs: directories: testdata: path: "./testdata"
+
+	actions: {
+		build: engine.#Build & {
+			source: inputs.directories.testdata.contents
+		}
+
+		verify: engine.#Exec & {
+			input: build.output
+			args: ["sh", "-c", "test $(cat /dir/foo) = foobar"]
+		}
+	}
+}

tests/tasks/build/dockerfile_path.cue

@@ -0,0 +1,21 @@
+package testing
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	inputs: directories: testdata: path: "./testdata"
+
+	actions: {
+		build: engine.#Build & {
+			source: inputs.directories.testdata.contents
+			dockerfile: path: "./dockerfilepath/Dockerfile.custom"
+		}
+
+		verify: engine.#Exec & {
+			input: build.output
+			args: ["sh", "-c", "test $(cat /test) = dockerfilePath"]
+		}
+	}
+}

tests/tasks/build/image_config.cue

@@ -0,0 +1,26 @@
+package testing
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	inputs: directories: testdata: path: "./testdata"
+
+	actions: {
+		// FIXME: this doesn't test anything beside not crashing
+		build: engine.#Build & {
+			source: inputs.directories.testdata.contents
+			dockerfile: contents: """
+				FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+				ENV test foobar
+				CMD /test-cmd
+				"""
+		} & {
+			config: {
+				Env: ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "test=foobar"]
+				Cmd: ["/bin/sh", "-c", "/test-cmd"]
+			}
+		}
+	}
+}

tests/tasks/build/inlined_dockerfile.cue

@@ -0,0 +1,94 @@
+package testing
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	inputs: directories: testdata: path: "./testdata"
+
+	actions: {
+		build: engine.#Build & {
+			source: inputs.directories.testdata.contents
+			dockerfile: contents: """
+				FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+				RUN echo foobar > /output
+				"""
+		}
+
+		verify: engine.#Exec & {
+			input: build.output
+			args: ["sh", "-c", "test $(cat /output) = foobar"]
+		}
+	}
+}
+
+// TestDockerfilePath: #up: [
+//  op.#DockerBuild & {
+//   context:        TestData
+//   dockerfilePath: "./dockerfilepath/Dockerfile.custom"
+//  },
+//  op.#Exec & {
+//   args: ["sh", "-c", "test $(cat /test) = dockerfilePath"]
+//  },
+// ]
+
+// TestBuildArgs: #up: [
+//  op.#DockerBuild & {
+//   dockerfile: """
+//    FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+//    ARG TEST=foo
+//    RUN test "${TEST}" = "bar"
+//    """
+//   buildArg: TEST: "bar"
+//  },
+// ]
+
+// // FIXME: this doesn't test anything beside not crashing
+// TestBuildLabels: #up: [
+//  op.#DockerBuild & {
+//   dockerfile: """
+//    FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+//    """
+//   label: FOO: "bar"
+//  },
+// ]
+
+// // FIXME: this doesn't test anything beside not crashing
+// TestBuildPlatform: #up: [
+//  op.#DockerBuild & {
+//   dockerfile: """
+//    FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+//    """
+//   platforms: ["linux/amd64"]
+//  },
+// ]
+
+// TestImageMetadata: #up: [
+//  op.#DockerBuild & {
+//   dockerfile: """
+//    FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+//    ENV CHECK foobar
+//    ENV DOUBLECHECK test
+//    """
+//  },
+//  op.#Exec & {
+//   args: ["sh", "-c", #"""
+//    env
+//    test "$CHECK" = "foobar"
+//    """#]
+//  },
+// ]
+
+// // Make sure the metadata is carried over with a `Load`
+// TestImageMetadataIndirect: #up: [
+//  op.#Load & {
+//   from: TestImageMetadata
+//  },
+//  op.#Exec & {
+//   args: ["sh", "-c", #"""
+//    env
+//    test "$DOUBLECHECK" = "test"
+//    """#]
+//  },
+// ]

tests/tasks/build/labels.cue

@@ -0,0 +1,20 @@
+package testing
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	inputs: directories: testdata: path: "./testdata"
+
+	actions: {
+		// FIXME: this doesn't test anything beside not crashing
+		build: engine.#Build & {
+			source: inputs.directories.testdata.contents
+			dockerfile: contents: """
+				FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+				"""
+			label: FOO: "bar"
+		}
+	}
+}

tests/tasks/build/platform.cue

@@ -0,0 +1,20 @@
+package testing
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	inputs: directories: testdata: path: "./testdata"
+
+	actions: {
+		// FIXME: this doesn't test anything beside not crashing
+		build: engine.#Build & {
+			source: inputs.directories.testdata.contents
+			dockerfile: contents: """
+				FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+				"""
+			platforms: ["linux/amd64"]
+		}
+	}
+}

tests/tasks/build/testdata/Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+COPY . /dir
+RUN test $(cat /dir/foo) = foobar

tests/tasks/build/testdata/dockerfilepath/Dockerfile.custom

@@ -0,0 +1,2 @@
+FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d
+RUN echo dockerfilePath > /test

tests/tasks/build/testdata/foo

@@ -0,0 +1 @@
+foobar