From bc2dae7e32cd27224579e360fb9173fe47079c4a Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Fri, 19 Feb 2021 00:09:53 -0800 Subject: [PATCH] docker build support Signed-off-by: Andrea Luzzardi --- Dockerfile | 2 +- dagger/fs.go | 9 ++ dagger/pipeline.go | 137 ++++++++++++++++++ dagger/solver.go | 26 ++-- examples/dogfood/main.cue | 15 ++ stdlib/dagger/dagger.cue | 12 ++ tests/dockerbuild/main.cue | 90 ++++++++++++ tests/dockerbuild/testdata/Dockerfile | 3 + .../testdata/dockerfilepath/Dockerfile.custom | 2 + tests/dockerbuild/testdata/foo | 1 + tests/test.sh | 6 + 11 files changed, 292 insertions(+), 11 deletions(-) create mode 100644 tests/dockerbuild/main.cue create mode 100644 tests/dockerbuild/testdata/Dockerfile create mode 100644 tests/dockerbuild/testdata/dockerfilepath/Dockerfile.custom create mode 100644 tests/dockerbuild/testdata/foo diff --git a/Dockerfile b/Dockerfile index c287a234..f6970859 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# syntax = docker/dockerfile-upstream:experimental@sha256:398a0a10f19875add7fe359a37f2f971c46746b064faf876776ae632a3472c37 +# syntax = docker/dockerfile:1.2 FROM golang:1.16-alpine AS build WORKDIR /src diff --git a/dagger/fs.go b/dagger/fs.go index 4205365e..e2e75a63 100644 --- a/dagger/fs.go +++ b/dagger/fs.go @@ -9,6 +9,7 @@ import ( "github.com/moby/buildkit/client/llb" bkgw "github.com/moby/buildkit/frontend/gateway/client" + bkpb "github.com/moby/buildkit/solver/pb" fstypes "github.com/tonistiigi/fsutil/types" "dagger.io/go/dagger/compiler" @@ -162,6 +163,14 @@ func (fs FS) LLB() llb.State { return fs.input } +func (fs FS) Def(ctx context.Context) (*bkpb.Definition, error) { + def, err := fs.LLB().Marshal(ctx, llb.LinuxAmd64) + if err != nil { + return nil, err + } + return def.ToPB(), nil +} + func (fs FS) Ref(ctx context.Context) (bkgw.Reference, error) { if err := (&fs).solve(ctx); err != nil { return nil, err diff --git a/dagger/pipeline.go b/dagger/pipeline.go index a55d614b..799ee34b 100644 --- a/dagger/pipeline.go +++ b/dagger/pipeline.go @@ -3,9 +3,14 @@ package dagger import ( "context" "encoding/json" + "errors" "fmt" + "strings" "github.com/moby/buildkit/client/llb" + dockerfilebuilder "github.com/moby/buildkit/frontend/dockerfile/builder" + bkgw "github.com/moby/buildkit/frontend/gateway/client" + bkpb "github.com/moby/buildkit/solver/pb" "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" @@ -157,6 +162,8 @@ func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value) error { return p.Load(ctx, op) case "subdir": return p.Subdir(ctx, op) + case "docker-build": + return p.DockerBuild(ctx, op) default: return fmt.Errorf("invalid operation: %s", op.JSON()) } @@ -431,6 +438,7 @@ func (p *Pipeline) Load(ctx context.Context, op *compiler.Value) error { if err := from.Do(ctx, op.Get("from")); err != nil { return err } + p.fs = p.fs.Set(from.FS().LLB()) return nil } @@ -457,3 +465,132 @@ func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value) error { p.fs = p.fs.Set(llb.Git(remote, ref)) return nil } + +func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error { + var ( + context = op.Lookup("context") + dockerfile = op.Lookup("dockerfile") + + contextDef *bkpb.Definition + dockerfileDef *bkpb.Definition + + err error + ) + + if !context.Exists() && !dockerfile.Exists() { + return errors.New("context or dockerfile required") + } + + // docker build context. This can come from another component, so we need to + // compute it first. + if context.Exists() { + from := p.Tmp() + if err := from.Do(ctx, context); err != nil { + return err + } + contextDef, err = from.FS().Def(ctx) + if err != nil { + return err + } + dockerfileDef = contextDef + } + + // Inlined dockerfile: need to be converted to LLB + if dockerfile.Exists() { + content, err := dockerfile.String() + if err != nil { + return err + } + dockerfileDef, err = p.s.Scratch().Set( + llb.Scratch().File( + llb.Mkfile("/Dockerfile", 0644, []byte(content)), + ), + ).Def(ctx) + if err != nil { + return err + } + if contextDef == nil { + contextDef = dockerfileDef + } + } + + req := bkgw.SolveRequest{ + Frontend: "dockerfile.v0", + FrontendOpt: make(map[string]string), + FrontendInputs: map[string]*bkpb.Definition{ + dockerfilebuilder.DefaultLocalNameContext: contextDef, + dockerfilebuilder.DefaultLocalNameDockerfile: dockerfileDef, + }, + } + + if dockerfilePath := op.Lookup("dockerfilePath"); dockerfilePath.Exists() { + filename, err := dockerfilePath.String() + if err != nil { + return err + } + req.FrontendOpt["filename"] = filename + } + + if buildArgs := op.Lookup("buildArg"); buildArgs.Exists() { + err := buildArgs.RangeStruct(func(key string, value *compiler.Value) error { + v, err := value.String() + if err != nil { + return err + } + req.FrontendOpt["build-arg:"+key] = v + return nil + }) + if err != nil { + return err + } + } + + if labels := op.Lookup("label"); labels.Exists() { + err := labels.RangeStruct(func(key string, value *compiler.Value) error { + s, err := value.String() + if err != nil { + return err + } + req.FrontendOpt["label:"+key] = s + return nil + }) + if err != nil { + return err + } + } + + if platforms := op.Lookup("platforms"); platforms.Exists() { + p := []string{} + list, err := platforms.List() + if err != nil { + return err + } + + for _, platform := range list { + s, err := platform.String() + if err != nil { + return err + } + p = append(p, s) + } + + if len(p) > 0 { + req.FrontendOpt["platform"] = strings.Join(p, ",") + } + if len(p) > 1 { + req.FrontendOpt["multi-platform"] = "true" + } + } + + res, err := p.s.SolveRequest(ctx, req) + if err != nil { + return err + } + st, err := res.ToState() + if err != nil { + return err + } + p.fs = p.fs.Set(st) + + return nil +} diff --git a/dagger/solver.go b/dagger/solver.go index 6d49bc49..eddac981 100644 --- a/dagger/solver.go +++ b/dagger/solver.go @@ -7,7 +7,7 @@ import ( "github.com/moby/buildkit/client/llb" bkgw "github.com/moby/buildkit/frontend/gateway/client" - "github.com/moby/buildkit/solver/pb" + bkpb "github.com/moby/buildkit/solver/pb" "github.com/opencontainers/go-digest" "github.com/rs/zerolog/log" ) @@ -35,6 +35,17 @@ func (s Solver) Scratch() FS { return s.FS(llb.Scratch()) } +// Solve will block until the state is solved and returns a Reference. +func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (bkgw.Reference, error) { + // call solve + res, err := s.c.Solve(ctx, req) + if err != nil { + return nil, bkCleanError(err) + } + // always use single reference (ignore multiple outputs & metadata) + return res.SingleRef() +} + // Solve will block until the state is solved and returns a Reference. func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) { // marshal llb @@ -55,7 +66,7 @@ func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) Msg("solving") // call solve - res, err := s.c.Solve(ctx, bkgw.SolveRequest{ + return s.SolveRequest(ctx, bkgw.SolveRequest{ Definition: def.ToPB(), // makes Solve() to block until LLB graph is solved. otherwise it will @@ -63,23 +74,18 @@ func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) // will be evaluated on export or if you access files on it. Evaluate: true, }) - if err != nil { - return nil, bkCleanError(err) - } - // always use single reference (ignore multiple outputs & metadata) - return res.SingleRef() } type llbOp struct { - Op pb.Op + Op bkpb.Op Digest digest.Digest - OpMetadata pb.OpMetadata + OpMetadata bkpb.OpMetadata } func dumpLLB(def *llb.Definition) ([]byte, error) { ops := make([]llbOp, 0, len(def.Def)) for _, dt := range def.Def { - var op pb.Op + var op bkpb.Op if err := (&op).Unmarshal(dt); err != nil { return nil, fmt.Errorf("failed to parse op: %w", err) } diff --git a/examples/dogfood/main.cue b/examples/dogfood/main.cue index 486a3c03..0e3b0505 100644 --- a/examples/dogfood/main.cue +++ b/examples/dogfood/main.cue @@ -7,6 +7,7 @@ import ( repository: dagger.#Dir // Use `--input-dir repository=.` from the root directory of the project +// Build `dagger` using Go build: go.#Build & { source: repository packages: "./cmd/dagger" @@ -18,7 +19,21 @@ test: go.#Test & { packages: "./..." } +// Run a command with the binary we just built help: #dagger: compute: [ dagger.#Load & {from: build}, dagger.#Exec & {args: ["dagger", "-h"]}, ] + +// Build dagger using the (included) Dockerfile +buildWithDocker: #dagger: compute: [ + dagger.#DockerBuild & { + context: repository + }, +] + +// Run a command in the docker image we just built +helpFromDocker: #dagger: compute: [ + dagger.#Load & {from: buildWithDocker}, + dagger.#Exec & {args: ["dagger", "-h"]}, +] diff --git a/stdlib/dagger/dagger.cue b/stdlib/dagger/dagger.cue index ad967e60..ecdc79c5 100644 --- a/stdlib/dagger/dagger.cue +++ b/stdlib/dagger/dagger.cue @@ -59,3 +59,15 @@ package dagger src: string | *"/" dest: string | *"/" } + +#DockerBuild: { + do: "docker-build" + // We accept either a context, a Dockerfile or both together + context?: _ + dockerfilePath?: string // path to the Dockerfile (defaults to "Dockerfile") + dockerfile?: string + + platforms?: [...string] + buildArg?: [string]: string + label?: [string]: string +} diff --git a/tests/dockerbuild/main.cue b/tests/dockerbuild/main.cue new file mode 100644 index 00000000..12d4c639 --- /dev/null +++ b/tests/dockerbuild/main.cue @@ -0,0 +1,90 @@ +package test + +import "dagger.io/dagger" + +// Set to `--input-dir=./tests/dockerbuild/testdata` +TestData: dagger.#Dir + +TestInlinedDockerfile: #dagger: compute: [ + dagger.#DockerBuild & { + dockerfile: """ + FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d + RUN echo hello world + """ + }, +] + +TestOpChaining: #dagger: compute: [ + dagger.#DockerBuild & { + dockerfile: """ + FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d + RUN echo foobar > /output + """ + }, + dagger.#Exec & { + args: ["sh", "-c", "test $(cat /output) = foobar"] + } +] + +TestBuildContext: #dagger: compute: [ + dagger.#DockerBuild & { + context: TestData + }, + dagger.#Exec & { + args: ["sh", "-c", "test $(cat /dir/foo) = foobar"] + } +] + +TestBuildContextAndDockerfile: #dagger: compute: [ + dagger.#DockerBuild & { + context: TestData + dockerfile: """ + FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d + COPY foo /override + """ + }, + dagger.#Exec & { + args: ["sh", "-c", "test $(cat /override) = foobar"] + } +] + +TestDockerfilePath: #dagger: compute: [ + dagger.#DockerBuild & { + context: TestData + dockerfilePath: "./dockerfilepath/Dockerfile.custom" + }, + dagger.#Exec & { + args: ["sh", "-c", "test $(cat /test) = dockerfilePath"] + } +] + +TestBuildArgs: #dagger: compute: [ + dagger.#DockerBuild & { + dockerfile: """ + FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d + ARG TEST=foo + RUN test "${TEST}" = "bar" + """ + buildArg: TEST: "bar" + } +] + +// FIXME: this doesn't test anything beside not crashing +TestBuildLabels: #dagger: compute: [ + dagger.#DockerBuild & { + dockerfile: """ + FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d + """ + label: FOO: "bar" + } +] + +// FIXME: this doesn't test anything beside not crashing +TestBuildPlatform: #dagger: compute: [ + dagger.#DockerBuild & { + dockerfile: """ + FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d + """ + platforms: ["linux/amd64"] + } +] diff --git a/tests/dockerbuild/testdata/Dockerfile b/tests/dockerbuild/testdata/Dockerfile new file mode 100644 index 00000000..6c3408cd --- /dev/null +++ b/tests/dockerbuild/testdata/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d +COPY . /dir +RUN test $(cat /dir/foo) = foobar diff --git a/tests/dockerbuild/testdata/dockerfilepath/Dockerfile.custom b/tests/dockerbuild/testdata/dockerfilepath/Dockerfile.custom new file mode 100644 index 00000000..d3a60705 --- /dev/null +++ b/tests/dockerbuild/testdata/dockerfilepath/Dockerfile.custom @@ -0,0 +1,2 @@ +FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d +RUN echo dockerfilePath > /test diff --git a/tests/dockerbuild/testdata/foo b/tests/dockerbuild/testdata/foo new file mode 100644 index 00000000..323fae03 --- /dev/null +++ b/tests/dockerbuild/testdata/foo @@ -0,0 +1 @@ +foobar diff --git a/tests/test.sh b/tests/test.sh index 93788624..3866bba4 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -229,6 +229,11 @@ test::subdir() { "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/subdir/simple } +test::dockerbuild() { + test::one "Docker Build" --exit=0 \ + "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute --input-dir TestData="$d"/dockerbuild/testdata "$d"/dockerbuild +} + test::all(){ local dagger="$1" @@ -244,6 +249,7 @@ test::all(){ test::export "$dagger" test::input "$dagger" test::subdir "$dagger" + test::dockerbuild "$dagger" test::examples "$dagger" }