From 2467fb1920535c420e6da39e8853d8c66f0bd40e Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Mon, 20 Dec 2021 19:52:22 +0100 Subject: [PATCH] engine.#Build support Signed-off-by: Andrea Luzzardi --- plan/task/build.go | 219 ++++++++++++++++++ stdlib/europa/dagger/engine/image.cue | 8 +- tests/tasks.bats | 15 +- tests/tasks/build/build_args.cue | 19 ++ tests/tasks/build/cue.mod/module.cue | 1 + tests/tasks/build/cue.mod/pkg/.gitignore | 3 + tests/tasks/build/dockerfile.cue | 20 ++ tests/tasks/build/dockerfile_path.cue | 21 ++ tests/tasks/build/image_config.cue | 26 +++ tests/tasks/build/inlined_dockerfile.cue | 94 ++++++++ tests/tasks/build/labels.cue | 20 ++ tests/tasks/build/platform.cue | 20 ++ tests/tasks/build/testdata/Dockerfile | 3 + .../testdata/dockerfilepath/Dockerfile.custom | 2 + tests/tasks/build/testdata/foo | 1 + 15 files changed, 469 insertions(+), 3 deletions(-) create mode 100644 plan/task/build.go create mode 100644 tests/tasks/build/build_args.cue create mode 100644 tests/tasks/build/cue.mod/module.cue create mode 100644 tests/tasks/build/cue.mod/pkg/.gitignore create mode 100644 tests/tasks/build/dockerfile.cue create mode 100644 tests/tasks/build/dockerfile_path.cue create mode 100644 tests/tasks/build/image_config.cue create mode 100644 tests/tasks/build/inlined_dockerfile.cue create mode 100644 tests/tasks/build/labels.cue create mode 100644 tests/tasks/build/platform.cue create mode 100644 tests/tasks/build/testdata/Dockerfile create mode 100644 tests/tasks/build/testdata/dockerfilepath/Dockerfile.custom create mode 100644 tests/tasks/build/testdata/foo diff --git a/plan/task/build.go b/plan/task/build.go new file mode 100644 index 00000000..7cde2082 --- /dev/null +++ b/plan/task/build.go @@ -0,0 +1,219 @@ +package task + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "cuelang.org/go/cue" + bkplatforms "github.com/containerd/containerd/platforms" + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + dockerfilebuilder "github.com/moby/buildkit/frontend/dockerfile/builder" + "github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb" + bkgw "github.com/moby/buildkit/frontend/gateway/client" + bkpb "github.com/moby/buildkit/solver/pb" + + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("Build", func() Task { return &buildTask{} }) +} + +type buildTask struct { +} + +func (t *buildTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + frontend, err := v.Lookup("frontend").String() + if err != nil { + return nil, err + } + + switch frontend { + case "dockerfile": + return t.dockerfile(ctx, pctx, s, v) + default: + return nil, fmt.Errorf("unsupported frontend %q", frontend) + } +} + +func (t *buildTask) dockerfile(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + // FIXME: support auth + + source, err := pctx.FS.FromValue(v.Lookup("source")) + if err != nil { + return nil, err + } + + sourceSt, err := source.Result().ToState() + if err != nil { + return nil, err + } + + // docker build context + contextDef, err := s.Marshal(ctx, sourceSt) + if err != nil { + return nil, err + } + // Dockerfile context, default to docker build context + dockerfileDef := contextDef + + // Support inlined dockerfile + if dockerfile := v.Lookup("dockerfile.contents"); dockerfile.Exists() { + contents, err := dockerfile.String() + if err != nil { + return nil, err + } + dockerfileDef, err = s.Marshal(ctx, + llb.Scratch().File( + llb.Mkfile("/Dockerfile", 0644, []byte(contents)), + ), + ) + if err != nil { + return nil, err + } + } + + opts, err := t.dockerBuildOpts(v, pctx) + if err != nil { + return nil, err + } + // Handle --no-cache + if s.NoCache() { + opts["no-cache"] = "" + } + + req := bkgw.SolveRequest{ + Frontend: "dockerfile.v0", + FrontendOpt: opts, + FrontendInputs: map[string]*bkpb.Definition{ + dockerfilebuilder.DefaultLocalNameContext: contextDef, + dockerfilebuilder.DefaultLocalNameDockerfile: dockerfileDef, + }, + } + res, err := s.SolveRequest(ctx, req) + if err != nil { + return nil, err + } + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + + out := compiler.NewValue() + if err := out.FillPath(cue.ParsePath("output"), pctx.FS.New(ref).MarshalCUE()); err != nil { + return nil, err + } + + // Load image metadata + if meta, ok := res.Metadata[exptypes.ExporterImageConfigKey]; ok { + var image dockerfile2llb.Image + if err := json.Unmarshal(meta, &image); err != nil { + return nil, fmt.Errorf("failed to unmarshal image config: %w", err) + } + if err := out.FillPath(cue.ParsePath("config"), image.Config); err != nil { + return nil, err + } + } + + return out, nil +} + +func (t *buildTask) dockerBuildOpts(v *compiler.Value, pctx *plancontext.Context) (map[string]string, error) { + opts := map[string]string{} + + if dockerfilePath := v.Lookup("dockerfile.path"); dockerfilePath.Exists() { + filename, err := dockerfilePath.String() + if err != nil { + return nil, err + } + opts["filename"] = filename + } + + if target := v.Lookup("target"); target.Exists() { + tgr, err := target.String() + if err != nil { + return nil, err + } + opts["target"] = tgr + } + + if hosts := v.Lookup("hosts"); hosts.Exists() { + p := []string{} + fields, err := hosts.Fields() + if err != nil { + return nil, err + } + for _, host := range fields { + s, err := host.Value.String() + if err != nil { + return nil, err + } + p = append(p, host.Label()+"="+s) + } + if len(p) > 0 { + opts["add-hosts"] = strings.Join(p, ",") + } + } + + if buildArgs := v.Lookup("buildArg"); buildArgs.Exists() { + fields, err := buildArgs.Fields() + if err != nil { + return nil, err + } + for _, buildArg := range fields { + s, err := buildArg.Value.String() + if err != nil { + return nil, err + } + opts["build-arg:"+buildArg.Label()] = s + } + } + + if labels := v.Lookup("label"); labels.Exists() { + fields, err := labels.Fields() + if err != nil { + return nil, err + } + for _, label := range fields { + s, err := label.Value.String() + if err != nil { + return nil, err + } + opts["label:"+label.Label()] = s + } + } + + if platforms := v.Lookup("platforms"); platforms.Exists() { + p := []string{} + list, err := platforms.List() + if err != nil { + return nil, err + } + + for _, platform := range list { + s, err := platform.String() + if err != nil { + return nil, err + } + p = append(p, s) + } + + if len(p) > 0 { + opts["platform"] = strings.Join(p, ",") + } + if len(p) > 1 { + opts["multi-platform"] = "true" + } + } + // Set platform to configured one if no one is defined + if opts["platform"] == "" { + opts["platform"] = bkplatforms.Format(pctx.Platform.Get()) + } + + return opts, nil +} diff --git a/stdlib/europa/dagger/engine/image.cue b/stdlib/europa/dagger/engine/image.cue index 13e0fbd6..67a725ec 100644 --- a/stdlib/europa/dagger/engine/image.cue +++ b/stdlib/europa/dagger/engine/image.cue @@ -71,7 +71,6 @@ package engine // Build a container image using buildkit // FIXME: rename to #Dockerfile to clarify scope #Build: { - @dagger(notimplemented) $dagger: task: _name: "Build" // Source directory to build @@ -83,6 +82,13 @@ package engine } | { contents: string } + + platforms: [...string] + target: string + buildArg: [string]: string + label: [string]: string + target: string + hosts: [string]: string } // Root filesystem produced by build diff --git a/tests/tasks.bats b/tests/tasks.bats index 08eaf466..db6d14b3 100644 --- a/tests/tasks.bats +++ b/tests/tasks.bats @@ -60,7 +60,6 @@ setup() { assert_failure } - @test "task: #Mkdir" { # Make directory cd "$TESTDIR"/tasks/mkdir @@ -74,4 +73,16 @@ setup() { cd "$TESTDIR"/tasks/mkdir run "$DAGGER" --europa up ./mkdir_failure_disable_parents.cue assert_failure -} \ No newline at end of file +} + +@test "task: #Build" { + cd "$TESTDIR"/tasks/build + + "$DAGGER" --europa up ./dockerfile.cue + "$DAGGER" --europa up ./inlined_dockerfile.cue + "$DAGGER" --europa up ./dockerfile_path.cue + "$DAGGER" --europa up ./build_args.cue + "$DAGGER" --europa up ./image_config.cue + "$DAGGER" --europa up ./labels.cue + "$DAGGER" --europa up ./platform.cue +} diff --git a/tests/tasks/build/build_args.cue b/tests/tasks/build/build_args.cue new file mode 100644 index 00000000..c4015e9a --- /dev/null +++ b/tests/tasks/build/build_args.cue @@ -0,0 +1,19 @@ +package testing + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + inputs: directories: testdata: path: "./testdata" + + actions: build: engine.#Build & { + source: inputs.directories.testdata.contents + dockerfile: contents: """ + FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d + ARG TEST=foo + RUN test "${TEST}" = "bar" + """ + buildArg: TEST: "bar" + } +} diff --git a/tests/tasks/build/cue.mod/module.cue b/tests/tasks/build/cue.mod/module.cue new file mode 100644 index 00000000..f8af9cef --- /dev/null +++ b/tests/tasks/build/cue.mod/module.cue @@ -0,0 +1 @@ +module: "" diff --git a/tests/tasks/build/cue.mod/pkg/.gitignore b/tests/tasks/build/cue.mod/pkg/.gitignore new file mode 100644 index 00000000..2d4dc1ae --- /dev/null +++ b/tests/tasks/build/cue.mod/pkg/.gitignore @@ -0,0 +1,3 @@ +# generated by dagger +alpha.dagger.io +dagger.lock diff --git a/tests/tasks/build/dockerfile.cue b/tests/tasks/build/dockerfile.cue new file mode 100644 index 00000000..6f044846 --- /dev/null +++ b/tests/tasks/build/dockerfile.cue @@ -0,0 +1,20 @@ +package testing + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + inputs: directories: testdata: path: "./testdata" + + actions: { + build: engine.#Build & { + source: inputs.directories.testdata.contents + } + + verify: engine.#Exec & { + input: build.output + args: ["sh", "-c", "test $(cat /dir/foo) = foobar"] + } + } +} diff --git a/tests/tasks/build/dockerfile_path.cue b/tests/tasks/build/dockerfile_path.cue new file mode 100644 index 00000000..2a3a6b13 --- /dev/null +++ b/tests/tasks/build/dockerfile_path.cue @@ -0,0 +1,21 @@ +package testing + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + inputs: directories: testdata: path: "./testdata" + + actions: { + build: engine.#Build & { + source: inputs.directories.testdata.contents + dockerfile: path: "./dockerfilepath/Dockerfile.custom" + } + + verify: engine.#Exec & { + input: build.output + args: ["sh", "-c", "test $(cat /test) = dockerfilePath"] + } + } +} diff --git a/tests/tasks/build/image_config.cue b/tests/tasks/build/image_config.cue new file mode 100644 index 00000000..28c8ebf3 --- /dev/null +++ b/tests/tasks/build/image_config.cue @@ -0,0 +1,26 @@ +package testing + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + inputs: directories: testdata: path: "./testdata" + + actions: { + // FIXME: this doesn't test anything beside not crashing + build: engine.#Build & { + source: inputs.directories.testdata.contents + dockerfile: contents: """ + FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d + ENV test foobar + CMD /test-cmd + """ + } & { + config: { + Env: ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "test=foobar"] + Cmd: ["/bin/sh", "-c", "/test-cmd"] + } + } + } +} diff --git a/tests/tasks/build/inlined_dockerfile.cue b/tests/tasks/build/inlined_dockerfile.cue new file mode 100644 index 00000000..feaae27d --- /dev/null +++ b/tests/tasks/build/inlined_dockerfile.cue @@ -0,0 +1,94 @@ +package testing + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + inputs: directories: testdata: path: "./testdata" + + actions: { + build: engine.#Build & { + source: inputs.directories.testdata.contents + dockerfile: contents: """ + FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d + RUN echo foobar > /output + """ + } + + verify: engine.#Exec & { + input: build.output + args: ["sh", "-c", "test $(cat /output) = foobar"] + } + } +} + +// TestDockerfilePath: #up: [ +// op.#DockerBuild & { +// context: TestData +// dockerfilePath: "./dockerfilepath/Dockerfile.custom" +// }, +// op.#Exec & { +// args: ["sh", "-c", "test $(cat /test) = dockerfilePath"] +// }, +// ] + +// TestBuildArgs: #up: [ +// op.#DockerBuild & { +// dockerfile: """ +// FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d +// ARG TEST=foo +// RUN test "${TEST}" = "bar" +// """ +// buildArg: TEST: "bar" +// }, +// ] + +// // FIXME: this doesn't test anything beside not crashing +// TestBuildLabels: #up: [ +// op.#DockerBuild & { +// dockerfile: """ +// FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d +// """ +// label: FOO: "bar" +// }, +// ] + +// // FIXME: this doesn't test anything beside not crashing +// TestBuildPlatform: #up: [ +// op.#DockerBuild & { +// dockerfile: """ +// FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d +// """ +// platforms: ["linux/amd64"] +// }, +// ] + +// TestImageMetadata: #up: [ +// op.#DockerBuild & { +// dockerfile: """ +// FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d +// ENV CHECK foobar +// ENV DOUBLECHECK test +// """ +// }, +// op.#Exec & { +// args: ["sh", "-c", #""" +// env +// test "$CHECK" = "foobar" +// """#] +// }, +// ] + +// // Make sure the metadata is carried over with a `Load` +// TestImageMetadataIndirect: #up: [ +// op.#Load & { +// from: TestImageMetadata +// }, +// op.#Exec & { +// args: ["sh", "-c", #""" +// env +// test "$DOUBLECHECK" = "test" +// """#] +// }, +// ] diff --git a/tests/tasks/build/labels.cue b/tests/tasks/build/labels.cue new file mode 100644 index 00000000..9a578eae --- /dev/null +++ b/tests/tasks/build/labels.cue @@ -0,0 +1,20 @@ +package testing + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + inputs: directories: testdata: path: "./testdata" + + actions: { + // FIXME: this doesn't test anything beside not crashing + build: engine.#Build & { + source: inputs.directories.testdata.contents + dockerfile: contents: """ + FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d + """ + label: FOO: "bar" + } + } +} diff --git a/tests/tasks/build/platform.cue b/tests/tasks/build/platform.cue new file mode 100644 index 00000000..e4a59937 --- /dev/null +++ b/tests/tasks/build/platform.cue @@ -0,0 +1,20 @@ +package testing + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + inputs: directories: testdata: path: "./testdata" + + actions: { + // FIXME: this doesn't test anything beside not crashing + build: engine.#Build & { + source: inputs.directories.testdata.contents + dockerfile: contents: """ + FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d + """ + platforms: ["linux/amd64"] + } + } +} diff --git a/tests/tasks/build/testdata/Dockerfile b/tests/tasks/build/testdata/Dockerfile new file mode 100644 index 00000000..6c3408cd --- /dev/null +++ b/tests/tasks/build/testdata/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d +COPY . /dir +RUN test $(cat /dir/foo) = foobar diff --git a/tests/tasks/build/testdata/dockerfilepath/Dockerfile.custom b/tests/tasks/build/testdata/dockerfilepath/Dockerfile.custom new file mode 100644 index 00000000..d3a60705 --- /dev/null +++ b/tests/tasks/build/testdata/dockerfilepath/Dockerfile.custom @@ -0,0 +1,2 @@ +FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d +RUN echo dockerfilePath > /test diff --git a/tests/tasks/build/testdata/foo b/tests/tasks/build/testdata/foo new file mode 100644 index 00000000..323fae03 --- /dev/null +++ b/tests/tasks/build/testdata/foo @@ -0,0 +1 @@ +foobar