diff --git a/client/client.go b/client/client.go index 8cbc279d..9667c6f7 100644 --- a/client/client.go +++ b/client/client.go @@ -7,6 +7,7 @@ import ( "strings" "sync" + "github.com/containerd/containerd/platforms" "go.opentelemetry.io/otel" "golang.org/x/sync/errgroup" @@ -80,7 +81,7 @@ func (c *Client) Do(ctx context.Context, state *state.State, fn DoFunc) error { lg := log.Ctx(ctx) eg, gctx := errgroup.WithContext(ctx) - environment, err := environment.New(state) + env, err := environment.New(state) if err != nil { return err } @@ -96,7 +97,7 @@ func (c *Client) Do(ctx context.Context, state *state.State, fn DoFunc) error { // Spawn build function eg.Go(func() error { - return c.buildfn(gctx, state, environment, fn, events) + return c.buildfn(gctx, state, env, fn, events) }) return eg.Wait() @@ -200,7 +201,7 @@ func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment. llb.WithCustomName("[internal] serializing computed values"), ) - ref, err := s.Solve(ctx, st) + ref, err := s.Solve(ctx, st, platforms.DefaultSpec()) if err != nil { return nil, err } diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index 80243a9a..be5c7ddb 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -9,6 +9,8 @@ import ( "strings" "cuelang.org/go/cue" + "github.com/containerd/containerd/platforms" + specs "github.com/opencontainers/image-spec/specs-go/v1" "go.dagger.io/dagger/cmd/dagger/cmd/common" "go.dagger.io/dagger/cmd/dagger/logger" "go.dagger.io/dagger/compiler" @@ -41,8 +43,9 @@ var computeCmd = &cobra.Command{ doneCh := common.TrackCommand(ctx, cmd) st := &state.State{ - Name: "FIXME", - Path: args[0], + Name: "FIXME", + Architecture: platforms.Format(specs.Platform{OS: "linux", Architecture: "amd64"}), + Path: args[0], Plan: state.Plan{ Module: args[0], }, diff --git a/cmd/dagger/cmd/edit.go b/cmd/dagger/cmd/edit.go index 67481f80..8a7961c5 100644 --- a/cmd/dagger/cmd/edit.go +++ b/cmd/dagger/cmd/edit.go @@ -73,6 +73,7 @@ var editCmd = &cobra.Command{ lg.Fatal().Err(err).Msg("failed to decode file") } st.Name = newState.Name + st.Architecture = newState.Architecture st.Plan = newState.Plan st.Inputs = newState.Inputs diff --git a/cmd/dagger/cmd/new.go b/cmd/dagger/cmd/new.go index 3ab0b6b3..a658418a 100644 --- a/cmd/dagger/cmd/new.go +++ b/cmd/dagger/cmd/new.go @@ -34,7 +34,7 @@ var newCmd = &cobra.Command{ st, err := project.Create(ctx, name, state.Plan{ Package: viper.GetString("package"), - }) + }, viper.GetString("architecture")) if err != nil { lg.Fatal().Err(err).Msg("failed to create environment") @@ -46,6 +46,7 @@ var newCmd = &cobra.Command{ func init() { newCmd.Flags().StringP("package", "p", "", "references the name of the Cue package within the module to use as a plan. Default: defer to cue loader") + newCmd.Flags().StringP("architecture", "a", "", "architecture of the running pipeline. Default: host architecture") if err := viper.BindPFlags(newCmd.Flags()); err != nil { panic(err) } diff --git a/environment/environment.go b/environment/environment.go index 01aa6413..d6419824 100644 --- a/environment/environment.go +++ b/environment/environment.go @@ -7,6 +7,8 @@ import ( "cuelang.org/go/cue" cueflow "cuelang.org/go/tools/flow" + "github.com/containerd/containerd/platforms" + specs "github.com/opencontainers/image-spec/specs-go/v1" "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/solver" "go.dagger.io/dagger/state" @@ -137,7 +139,7 @@ func (e *Environment) Up(ctx context.Context, s solver.Solver) error { flow := cueflow.New( &cueflow.Config{}, e.src.Cue(), - newTaskFunc(newPipelineRunner(e.computed, s)), + newTaskFunc(newPipelineRunner(e.computed, s, e.state.Architecture)), ) if err := flow.Run(ctx); err != nil { return err @@ -176,7 +178,7 @@ func noOpRunner(t *cueflow.Task) error { return nil } -func newPipelineRunner(computed *compiler.Value, s solver.Solver) cueflow.RunnerFunc { +func newPipelineRunner(computed *compiler.Value, s solver.Solver, platform string) cueflow.RunnerFunc { return cueflow.RunnerFunc(func(t *cueflow.Task) error { ctx := t.Context() lg := log. @@ -197,7 +199,24 @@ func newPipelineRunner(computed *compiler.Value, s solver.Solver) cueflow.Runner Msg("dependency detected") } v := compiler.Wrap(t.Value()) - p := NewPipeline(v, s) + + var pipelinePlatform specs.Platform + if platform == "" { + pipelinePlatform = specs.Platform{OS: "linux", Architecture: "amd64"} + } else { + p, err := platforms.Parse(platform) + if err != nil { + // Record the error + span.AddEvent("command", trace.WithAttributes( + attribute.String("error", err.Error()), + )) + + return err + } + pipelinePlatform = p + } + + p := NewPipeline(v, s, pipelinePlatform) err := p.Run(ctx) if err != nil { // Record the error diff --git a/environment/pipeline.go b/environment/pipeline.go index 0fcdf0c9..85556c4b 100644 --- a/environment/pipeline.go +++ b/environment/pipeline.go @@ -13,6 +13,7 @@ import ( "time" "cuelang.org/go/cue" + bkplatforms "github.com/containerd/containerd/platforms" "github.com/docker/distribution/reference" bk "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" @@ -22,6 +23,7 @@ import ( bkgw "github.com/moby/buildkit/frontend/gateway/client" bkpb "github.com/moby/buildkit/solver/pb" digest "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" @@ -44,15 +46,17 @@ type Pipeline struct { name string s solver.Solver state llb.State + platform specs.Platform // Architecture constraint result bkgw.Reference image dockerfile2llb.Image computed *compiler.Value } -func NewPipeline(code *compiler.Value, s solver.Solver) *Pipeline { +func NewPipeline(code *compiler.Value, s solver.Solver, platform specs.Platform) *Pipeline { return &Pipeline{ code: code, name: code.Path().String(), + platform: platform, s: s, state: llb.Scratch(), computed: compiler.NewValue(), @@ -229,7 +233,7 @@ func (p *Pipeline) run(ctx context.Context) error { // so that errors map to the correct cue path. // FIXME: might as well change FS to make every operation // synchronous. - p.result, err = p.s.Solve(ctx, p.state) + p.result, err = p.s.Solve(ctx, p.state, p.platform) if err != nil { return err } @@ -335,7 +339,7 @@ func (p *Pipeline) Copy(ctx context.Context, op *compiler.Value, st llb.State) ( return st, err } // Execute 'from' in a tmp pipeline, and use the resulting fs - from := NewPipeline(op.Lookup("from"), p.s) + from := NewPipeline(op.Lookup("from"), p.s, p.platform) if err := from.Run(ctx); err != nil { return st, err } @@ -591,7 +595,7 @@ func (p *Pipeline) mount(ctx context.Context, dest string, mnt *compiler.Value) return nil, fmt.Errorf("invalid mount: should have %s structure", "{from: _, path: string | *\"/\"}") } - from := NewPipeline(mnt.Lookup("from"), p.s) + from := NewPipeline(mnt.Lookup("from"), p.s, p.platform) if err := from.Run(ctx); err != nil { return nil, err } @@ -737,7 +741,7 @@ func parseStringOrSecret(ctx context.Context, ss solver.SecretsStore, v *compile func (p *Pipeline) Load(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) { // Execute 'from' in a tmp pipeline, and use the resulting fs - from := NewPipeline(op.Lookup("from"), p.s) + from := NewPipeline(op.Lookup("from"), p.s, p.platform) if err := from.Run(ctx); err != nil { return st, err } @@ -795,7 +799,8 @@ func (p *Pipeline) FetchContainer(ctx context.Context, op *compiler.Value, st ll // Load image metadata and convert to to LLB. p.image, err = p.s.ResolveImageConfig(ctx, ref.String(), llb.ResolveImageConfigOpt{ - LogName: p.vertexNamef("load metadata for %s", ref.String()), + LogName: p.vertexNamef("load metadata for %s", ref.String()), + Platform: &p.platform, }) if err != nil { return st, err @@ -855,18 +860,18 @@ func (p *Pipeline) PushContainer(ctx context.Context, op *compiler.Value, st llb return st, err } - if digest, ok := resp.ExporterResponse["containerimage.digest"]; ok { + if dgst, ok := resp.ExporterResponse["containerimage.digest"]; ok { imageRef := fmt.Sprintf( "%s@%s", resp.ExporterResponse["image.name"], - digest, + dgst, ) return st.File( llb.Mkdir("/dagger", fs.FileMode(0755)), llb.WithCustomName(p.vertexNamef("Mkdir /dagger")), ).File( - llb.Mkfile("/dagger/image_digest", fs.FileMode(0644), []byte(digest)), + llb.Mkfile("/dagger/image_digest", fs.FileMode(0644), []byte(dgst)), llb.WithCustomName(p.vertexNamef("Storing image digest to /dagger/image_digest")), ).File( llb.Mkfile("/dagger/image_ref", fs.FileMode(0644), []byte(imageRef)), @@ -1068,7 +1073,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S // docker build context. This can come from another component, so we need to // compute it first. if dockerContext.Exists() { - from := NewPipeline(op.Lookup("context"), p.s) + from := NewPipeline(op.Lookup("context"), p.s, p.platform) if err := from.Run(ctx); err != nil { return st, err } @@ -1107,6 +1112,11 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S opts["no-cache"] = "" } + // Set platform to configured one if no one is defined + if opts["platform"] == "" { + opts["platform"] = bkplatforms.Format(p.platform) + } + req := bkgw.SolveRequest{ Frontend: "dockerfile.v0", FrontendOpt: opts, diff --git a/go.mod b/go.mod index f2c4b2f1..72c030a6 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( filippo.io/age v1.0.0 github.com/KromDaniel/jonson v0.0.0-20180630143114-d2f9c3c389db github.com/containerd/console v1.0.3 + github.com/containerd/containerd v1.5.4 // indirect github.com/docker/buildx v0.6.2 github.com/docker/distribution v2.7.1+incompatible github.com/emicklei/proto v1.9.0 // indirect @@ -21,6 +22,7 @@ require ( github.com/moby/buildkit v0.9.1 github.com/morikuni/aec v1.0.0 github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.0.1 // indirect github.com/rs/zerolog v1.23.0 github.com/spf13/cobra v1.2.1 github.com/spf13/viper v1.8.1 diff --git a/solver/solver.go b/solver/solver.go index 05f890ae..a3f6d7b3 100644 --- a/solver/solver.go +++ b/solver/solver.go @@ -16,6 +16,7 @@ import ( "github.com/moby/buildkit/session" bkpb "github.com/moby/buildkit/solver/pb" "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/rs/zerolog/log" ) @@ -79,9 +80,9 @@ func (s Solver) AddCredentials(target, username, secret string) { s.opts.Auth.AddCredentials(target, username, secret) } -func (s Solver) Marshal(ctx context.Context, st llb.State) (*bkpb.Definition, error) { +func (s Solver) Marshal(ctx context.Context, st llb.State, co ...llb.ConstraintsOpt) (*bkpb.Definition, error) { // FIXME: do not hardcode the platform - def, err := st.Marshal(ctx, llb.LinuxAmd64) + def, err := st.Marshal(ctx, co...) if err != nil { return nil, err } @@ -126,9 +127,9 @@ func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw. } // Solve will block until the state is solved and returns a Reference. -func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) { - // marshal llb - def, err := s.Marshal(ctx, st) +// It takes a platform as argument which correspond to the architecture. +func (s Solver) Solve(ctx context.Context, st llb.State, platform specs.Platform) (bkgw.Reference, error) { + def, err := s.Marshal(ctx, st, llb.Platform(platform)) if err != nil { return nil, err } diff --git a/state/project.go b/state/project.go index 6176def6..f74f5612 100644 --- a/state/project.go +++ b/state/project.go @@ -235,7 +235,7 @@ func (w *Project) Save(ctx context.Context, st *State) error { return nil } -func (w *Project) Create(ctx context.Context, name string, plan Plan) (*State, error) { +func (w *Project) Create(ctx context.Context, name string, plan Plan, arch string) (*State, error) { if _, err := w.Get(ctx, name); err == nil { return nil, ErrExist } @@ -263,7 +263,8 @@ func (w *Project) Create(ctx context.Context, name string, plan Plan) (*State, e Plan: Plan{ Package: pkg, }, - Name: name, + Name: name, + Architecture: arch, } data, err := yaml.Marshal(st) diff --git a/state/project_test.go b/state/project_test.go index 137a3333..7eeb7b93 100644 --- a/state/project_test.go +++ b/state/project_test.go @@ -32,9 +32,10 @@ func TestProject(t *testing.T) { // Create st, err := project.Create(ctx, "test", Plan{ Module: ".", - }) + }, "linux/amd64") require.NoError(t, err) require.Equal(t, "test", st.Name) + require.Equal(t, "linux/amd64", st.Architecture) // Open project, err = Open(ctx, root) @@ -51,6 +52,7 @@ func TestProject(t *testing.T) { env, err := project.Get(ctx, "test") require.NoError(t, err) require.Equal(t, "test", env.Name) + require.Equal(t, "linux/amd64", env.Architecture) // Save require.NoError(t, env.SetInput("foo", TextInput("bar"))) @@ -82,7 +84,7 @@ func TestEncryption(t *testing.T) { _, err = project.Create(ctx, "test", Plan{ Module: ".", - }) + }, "linux/amd64") require.NoError(t, err) // Set a plaintext input, make sure it is not encrypted diff --git a/state/state.go b/state/state.go index 63e1767d..75b38423 100644 --- a/state/state.go +++ b/state/state.go @@ -24,6 +24,9 @@ type State struct { // FIXME: store multiple names? Name string `yaml:"name,omitempty"` + // Architecture execution + Architecture string `yaml:"architecture,omitempty"` + // User Inputs Inputs map[string]Input `yaml:"inputs,omitempty"` diff --git a/tests/core.bats b/tests/core.bats index 64ba7ad4..9807e05b 100644 --- a/tests/core.bats +++ b/tests/core.bats @@ -159,6 +159,28 @@ setup() { "$DAGGER" up } +@test "core: arch config" { + dagger init + + # Test for amd64 architecture + dagger_new_with_plan test-amd "$TESTDIR"/core/arch-config "linux/amd64" + + # Set arch expected value + "$DAGGER" -e test-amd input text targetArch "x86_64" + + # Up amd + "$DAGGER" -e test-amd up + + # Test for amd64 architecture + dagger_new_with_plan test-arm "$TESTDIR"/core/arch-config "linux/arm64" + + # Set arch expected value + "$DAGGER" -e test-arm input text targetArch "aarch64" + + # Up arm + "$DAGGER" -e test-arm up +} + @test "compute: exclude" { "$DAGGER" up --project "$TESTDIR"/compute/exclude } diff --git a/tests/core/arch-config/arch-config.cue b/tests/core/arch-config/arch-config.cue new file mode 100644 index 00000000..23585932 --- /dev/null +++ b/tests/core/arch-config/arch-config.cue @@ -0,0 +1,58 @@ +package main + +import ( + "alpha.dagger.io/dagger/op" + "alpha.dagger.io/dagger" +) + +targetArch: dagger.#Input & {string} + +TestFetch: #up: [ + op.#FetchContainer & { + ref: "docker.io/alpine" + }, + + op.#Exec & { + args: ["/bin/sh", "-c", "echo $(uname -a) >> /arch.txt"] + always: true + }, + + op.#Exec & { + args: ["/bin/sh", "-c", """ + cat /arch.txt | grep "$TARGET_ARCH" + """] + env: TARGET_ARCH: targetArch + }, +] + +TestBuild: #up: [ + op.#DockerBuild & { + dockerfile: """ + FROM alpine + + RUN echo $(uname -a) > /arch.txt + """ + }, + + op.#Exec & { + args: ["/bin/sh", "-c", """ + cat /arch.txt | grep "$TARGET_ARCH" + """] + env: TARGET_ARCH: targetArch + }, +] + +TestLoad: #up: [ + op.#Load & { + from: TestBuild + }, + + // Compare arch + op.#Exec & { + args: ["/bin/sh", "-c", "diff /build/arch.txt /fetch/arch.txt"] + mount: { + "/build": from: TestBuild + "/fetch": from: TestFetch + } + }, +] diff --git a/tests/helpers.bash b/tests/helpers.bash index 5daf576f..5735963e 100644 --- a/tests/helpers.bash +++ b/tests/helpers.bash @@ -20,10 +20,19 @@ common_setup() { dagger_new_with_plan() { local name="$1" local sourcePlan="$2" + local arch="$3" cp -a "$sourcePlan"/* "$DAGGER_PROJECT" - "$DAGGER" new "$name" + local opts="" + if [ -n "$arch" ]; + then + opts="-a $arch" + fi + + # Need word splitting to take in account "-a" and "$arch" + # shellcheck disable=SC2086 + "$DAGGER" new "$name" ${opts} } dagger_new_with_env() {