From 2b641df5770756cb51ae0bed6506e56a99e9f900 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Sun, 30 Oct 2022 18:13:57 +0100 Subject: [PATCH] rework pipeline --- example/{ => golang-bin}/main.go | 14 ++- go.mod | 2 +- go.sum | 2 + pkg/cli/cli.go | 18 +++- pkg/pipelines/default.go | 23 +++++ pkg/pipelines/golang-bin.go | 119 ++++++++++++++++++++++++++ pkg/pipelines/pipeline.go | 44 ++++++++++ pkg/tasks/build.go | 44 ---------- pkg/tasks/container/create_scratch.go | 13 +++ pkg/tasks/container/load.go | 13 +++ pkg/tasks/container/mount.go | 27 ++++++ pkg/tasks/container/workdir.go | 9 ++ pkg/tasks/golang-bin/build.go | 29 +++++++ pkg/tasks/golang/cache.go | 18 ++++ pkg/tasks/golang/test.go | 22 +++++ 15 files changed, 345 insertions(+), 52 deletions(-) rename example/{ => golang-bin}/main.go (51%) create mode 100644 pkg/pipelines/default.go create mode 100644 pkg/pipelines/golang-bin.go create mode 100644 pkg/pipelines/pipeline.go delete mode 100644 pkg/tasks/build.go create mode 100644 pkg/tasks/container/create_scratch.go create mode 100644 pkg/tasks/container/load.go create mode 100644 pkg/tasks/container/mount.go create mode 100644 pkg/tasks/container/workdir.go create mode 100644 pkg/tasks/golang-bin/build.go create mode 100644 pkg/tasks/golang/cache.go create mode 100644 pkg/tasks/golang/test.go diff --git a/example/main.go b/example/golang-bin/main.go similarity index 51% rename from example/main.go rename to example/golang-bin/main.go index fbb6085..32f322d 100644 --- a/example/main.go +++ b/example/golang-bin/main.go @@ -5,7 +5,7 @@ import ( "log" "git.front.kjuulh.io/kjuulh/dagger-go/internal" - "git.front.kjuulh.io/kjuulh/dagger-go/pkg/tasks" + "git.front.kjuulh.io/kjuulh/dagger-go/pkg/pipelines" ) func main() { @@ -15,7 +15,6 @@ func main() { log.Fatal(err) } } - func run(ctx context.Context) error { builder, err := internal.New(ctx) if err != nil { @@ -23,5 +22,14 @@ func run(ctx context.Context) error { } defer builder.CleanUp() - return tasks.Build(builder, "some-image", "example/main.go") + return pipelines. + New(builder). + WithGolangBin(&pipelines.GolangBinOpts{ + DockerImageOpt: &pipelines.DockerImageOpt{ + ImageName: "golang-bin", + }, + BuildPath: "example/golang-bin/main.go", + BinName: "golang-bin", + }). + Execute(ctx) } diff --git a/go.mod b/go.mod index eb2f3f2..ff6efa3 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( ) require ( - git.front.kjuulh.io/kjuulh/byg v0.0.1 // indirect + git.front.kjuulh.io/kjuulh/byg v0.0.7 // indirect github.com/Khan/genqlient v0.5.0 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/agext/levenshtein v1.2.3 // indirect diff --git a/go.sum b/go.sum index b113fb4..8320105 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ dagger.io/dagger v0.3.1/go.mod h1:+p5s9Itrr/KT4UttGNpeUTNtVQUI2z9LDIOH9wBzu8g= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.front.kjuulh.io/kjuulh/byg v0.0.1 h1:SBrM5WBv1zTzt29pQMwGUJljGRI7Li7uATnZ3taotC0= git.front.kjuulh.io/kjuulh/byg v0.0.1/go.mod h1:8Vg5Mgqzva5fzHGeMlxjp/DMLbUy9uaKFxMNB34yYuA= +git.front.kjuulh.io/kjuulh/byg v0.0.7 h1:0ZDkRj1R2lvbWUQO5GjWJum9zpVMej6l8ZXn5YNmBNE= +git.front.kjuulh.io/kjuulh/byg v0.0.7/go.mod h1:8Vg5Mgqzva5fzHGeMlxjp/DMLbUy9uaKFxMNB34yYuA= github.com/99designs/gqlgen v0.17.2/go.mod h1:K5fzLKwtph+FFgh9j7nFbRUdBKvTcGnsta51fsMTn3o= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 725a879..3c595c5 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -7,7 +7,7 @@ import ( "os" "git.front.kjuulh.io/kjuulh/dagger-go/internal" - "git.front.kjuulh.io/kjuulh/dagger-go/pkg/tasks" + "git.front.kjuulh.io/kjuulh/dagger-go/pkg/pipelines" "github.com/spf13/cobra" ) @@ -31,13 +31,23 @@ func Build(mainGoPath string, imageTag string) *cobra.Command { log.Printf("Building image: %s\n", imageTag) - client, err := internal.New(ctx) + builder, err := internal.New(ctx) if err != nil { return err } - defer client.CleanUp() + defer builder.CleanUp() + + return pipelines. + New(builder). + WithGolangBin(&pipelines.GolangBinOpts{ + DockerImageOpt: &pipelines.DockerImageOpt{ + ImageName: "golang-bin", + }, + BuildPath: "example/golang-bin/main.go", + BinName: "golang-bin", + }). + Execute(ctx) - return tasks.Build(client, imageTag, mainGoPath) }, } diff --git a/pkg/pipelines/default.go b/pkg/pipelines/default.go new file mode 100644 index 0000000..9fad131 --- /dev/null +++ b/pkg/pipelines/default.go @@ -0,0 +1,23 @@ +package pipelines + +import ( + "context" + "log" + + "git.front.kjuulh.io/kjuulh/byg" +) + +func (p *Pipeline) WithDefault() error { + return byg. + New(). + Step( + "default step", + byg.Step{ + Execute: func(ctx byg.Context) error { + log.Println("Hello, world!") + return nil + }, + }). + Execute(context.Background()) + +} diff --git a/pkg/pipelines/golang-bin.go b/pkg/pipelines/golang-bin.go new file mode 100644 index 0000000..091a448 --- /dev/null +++ b/pkg/pipelines/golang-bin.go @@ -0,0 +1,119 @@ +package pipelines + +import ( + "context" + "fmt" + "log" + "strconv" + "time" + + "dagger.io/dagger" + "git.front.kjuulh.io/kjuulh/byg" + "git.front.kjuulh.io/kjuulh/dagger-go/pkg/tasks/container" + "git.front.kjuulh.io/kjuulh/dagger-go/pkg/tasks/golang" + golangbin "git.front.kjuulh.io/kjuulh/dagger-go/pkg/tasks/golang-bin" +) + +type DockerImageOpt struct { + ImageName string + ImageTag string +} + +type GolangBinOpts struct { + *DockerImageOpt + BuildPath string + BinName string +} + +func (p *Pipeline) WithGolangBin(opts *GolangBinOpts) *Pipeline { + log.Printf("building image: %s", opts.ImageName) + + client := p.builder.Dagger + ctx := context.Background() + + var ( + bin dagger.FileID + build *dagger.Container + scratch *dagger.Container + finalImage *dagger.Container + ) + + pipeline := byg. + New(). + Step( + "build golang", + byg.Step{ + Execute: func(_ byg.Context) error { + var err error + c := container.LoadImage(client, "harbor.front.kjuulh.io/docker-proxy/library/golang") + c, err = container.MountCurrent(ctx, client, c, "/src") + if err != nil { + return err + } + c = container.Workdir(c, "/src") + + build, err = golang.Cache(ctx, client, c) + if err != nil { + return err + } + + bin, err = golangbin.Build(ctx, build, opts.BinName, opts.BuildPath) + if err != nil { + return err + } + + return err + }, + }, + byg.Step{ + Execute: func(ctx byg.Context) error { + scratch = container.LoadImage(client, "harbor.front.kjuulh.io/docker-proxy/library/busybox") + return nil + }, + }, + ). + Step( + "create-production-image", + byg.Step{ + Execute: func(ctx byg.Context) error { + tempmount := fmt.Sprintf("/tmp/%s", opts.BinName) + usrbin := fmt.Sprintf("/usr/bin/%s", opts.BinName) + c := container.MountFileFromLoaded(scratch, bin, tempmount) + c = c.Exec(dagger.ContainerExecOpts{ + Args: []string{"mkdir", "-p", "/usr/bin"}, + }) + c = c.Exec(dagger.ContainerExecOpts{ + Args: []string{"cp", tempmount, usrbin}, + }) + finalImage = c.WithEntrypoint([]string{opts.BinName}) + + return nil + }, + }, + byg.Step{ + Execute: func(_ byg.Context) error { + return golang.Test(ctx, build) + }, + }, + ). + Step( + "upload-image", + byg.Step{ + Execute: func(_ byg.Context) error { + + if opts.ImageTag == "" { + opts.ImageTag = strconv.FormatInt(time.Now().UTC().UnixMilli(), 10) + } + + tag := fmt.Sprintf("harbor.front.kjuulh.io/kjuulh/%s:%s", opts.ImageName, opts.ImageTag) + + _, err := finalImage.Publish(ctx, tag) + return err + }, + }, + ) + + p.add(pipeline) + + return p +} diff --git a/pkg/pipelines/pipeline.go b/pkg/pipelines/pipeline.go new file mode 100644 index 0000000..7c7727c --- /dev/null +++ b/pkg/pipelines/pipeline.go @@ -0,0 +1,44 @@ +package pipelines + +import ( + "context" + + "git.front.kjuulh.io/kjuulh/byg" + "git.front.kjuulh.io/kjuulh/dagger-go/internal" + "golang.org/x/sync/errgroup" +) + +type Pipeline struct { + builder *internal.Builder + pipelines []*byg.Builder +} + +func New(builder *internal.Builder) *Pipeline { + return &Pipeline{builder: builder} +} + +func (p *Pipeline) WithCustom(custom func(p *Pipeline) *byg.Builder) { + p.add(custom(p)) +} + +func (p *Pipeline) Execute(ctx context.Context) error { + errgroup, ctx := errgroup.WithContext(ctx) + + for _, pipeline := range p.pipelines { + pipeline := pipeline // Allocate for closure + + errgroup.Go(func() error { + return pipeline.Execute(ctx) + }) + } + + if err := errgroup.Wait(); err != nil { + return err + } + + return nil +} + +func (p *Pipeline) add(pipeline *byg.Builder) { + p.pipelines = append(p.pipelines, pipeline) +} diff --git a/pkg/tasks/build.go b/pkg/tasks/build.go deleted file mode 100644 index 9d288fa..0000000 --- a/pkg/tasks/build.go +++ /dev/null @@ -1,44 +0,0 @@ -package tasks - -import ( - "context" - "log" - - "dagger.io/dagger" - "git.front.kjuulh.io/kjuulh/byg" - "git.front.kjuulh.io/kjuulh/dagger-go/internal" -) - -func Build(builder *internal.Builder, imageTag string, buildPath string) error { - log.Printf("building image: %s", imageTag) - - client := builder.Dagger - ctx := context.Background() - - return byg. - New(). - Step( - "build golang", - byg.Step{ - Execute: func(_ byg.Context) error { - src, err := client. - Host(). - Workdir(). - Read(). - ID(context.Background()) - if err != nil { - return err - } - - golang := client.Container().From("harbor.front.kjuulh.io/docker-proxy/library/golang:alpine") - golang = golang.WithMountedDirectory("/src", src).WithWorkdir("/src") - _, err = golang.Exec(dagger.ContainerExecOpts{ - Args: []string{"go", "build", "-o", "build/", buildPath}, - }).ExitCode(ctx) - - return err - }, - }). - Execute(context.Background()) - -} diff --git a/pkg/tasks/container/create_scratch.go b/pkg/tasks/container/create_scratch.go new file mode 100644 index 0000000..b558004 --- /dev/null +++ b/pkg/tasks/container/create_scratch.go @@ -0,0 +1,13 @@ +package container + +import ( + "log" + + "dagger.io/dagger" +) + +func CreateScratch(client *dagger.Client) *dagger.Container { + log.Println("creating scratch image") + + return client.Container(dagger.ContainerOpts{ID: ""}) +} diff --git a/pkg/tasks/container/load.go b/pkg/tasks/container/load.go new file mode 100644 index 0000000..820c7cb --- /dev/null +++ b/pkg/tasks/container/load.go @@ -0,0 +1,13 @@ +package container + +import ( + "log" + + "dagger.io/dagger" +) + +func LoadImage(client *dagger.Client, image string) *dagger.Container { + log.Printf("loading image: %s", image) + + return client.Container().From(image) +} diff --git a/pkg/tasks/container/mount.go b/pkg/tasks/container/mount.go new file mode 100644 index 0000000..b20d909 --- /dev/null +++ b/pkg/tasks/container/mount.go @@ -0,0 +1,27 @@ +package container + +import ( + "context" + "log" + + "dagger.io/dagger" +) + +func MountCurrent(ctx context.Context, client *dagger.Client, container *dagger.Container, into string) (*dagger.Container, error) { + log.Printf("mounting current working directory into path: %s", into) + src, err := client. + Host(). + Workdir(). + Read(). + ID(ctx) + if err != nil { + return nil, err + } + + return container.WithMountedDirectory(into, src), nil +} + +func MountFileFromLoaded(container *dagger.Container, bin dagger.FileID, path string) *dagger.Container { + log.Printf("mounting binary into container: into (path=%s)", path) + return container.WithMountedFile(path, bin) +} diff --git a/pkg/tasks/container/workdir.go b/pkg/tasks/container/workdir.go new file mode 100644 index 0000000..2135a4b --- /dev/null +++ b/pkg/tasks/container/workdir.go @@ -0,0 +1,9 @@ +package container + +import ( + "dagger.io/dagger" +) + +func Workdir(container *dagger.Container, into string) *dagger.Container { + return container.WithWorkdir(into) +} diff --git a/pkg/tasks/golang-bin/build.go b/pkg/tasks/golang-bin/build.go new file mode 100644 index 0000000..d0582cd --- /dev/null +++ b/pkg/tasks/golang-bin/build.go @@ -0,0 +1,29 @@ +package golangbin + +import ( + "context" + "fmt" + "log" + + "dagger.io/dagger" +) + +func Build(ctx context.Context, container *dagger.Container, binname string, buildpath string) (dagger.FileID, error) { + log.Printf("building binary: (binName=%s) into (buildPath=%s)", binname, buildpath) + binpath := fmt.Sprintf("dist/%s", binname) + c := container.Exec(dagger.ContainerExecOpts{ + Args: []string{"go", "build", "-o", binpath, buildpath}, + }) + + _, err := c.ExitCode(ctx) + if err != nil { + return "", err + } + + bin, err := c.File(binpath).ID(ctx) + if err != nil { + return "", err + } + + return bin, nil +} diff --git a/pkg/tasks/golang/cache.go b/pkg/tasks/golang/cache.go new file mode 100644 index 0000000..e4be66b --- /dev/null +++ b/pkg/tasks/golang/cache.go @@ -0,0 +1,18 @@ +package golang + +import ( + "context" + + "dagger.io/dagger" +) + +func Cache(ctx context.Context, client *dagger.Client, container *dagger.Container) (*dagger.Container, error) { + cacheKey := "gomods" + cacheID, err := client.CacheVolume(cacheKey).ID(ctx) + if err != nil { + return nil, err + } + return container. + WithMountedCache(cacheID, "/cache"). + WithEnvVariable("GOMODCACHE", "/cache"), nil +} diff --git a/pkg/tasks/golang/test.go b/pkg/tasks/golang/test.go new file mode 100644 index 0000000..03e27c1 --- /dev/null +++ b/pkg/tasks/golang/test.go @@ -0,0 +1,22 @@ +package golang + +import ( + "context" + "log" + + "dagger.io/dagger" +) + +func Test(ctx context.Context, container *dagger.Container) error { + log.Printf("testing: image") + c := container.Exec(dagger.ContainerExecOpts{ + Args: []string{"go", "test", "./..."}, + }) + + _, err := c.ExitCode(ctx) + if err != nil { + return err + } + + return nil +}