Merge pull request #1074 from TomChv/feat/pipeline-platform-config

Handle running architecture configuration

client/client.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/containerd/containerd/platforms"
 	"go.opentelemetry.io/otel"
 	"golang.org/x/sync/errgroup"
 
@@ -80,7 +81,7 @@ func (c *Client) Do(ctx context.Context, state *state.State, fn DoFunc) error {
 	lg := log.Ctx(ctx)
 	eg, gctx := errgroup.WithContext(ctx)
 
-	environment, err := environment.New(state)
+	env, err := environment.New(state)
 	if err != nil {
 		return err
 	}
@@ -96,7 +97,7 @@ func (c *Client) Do(ctx context.Context, state *state.State, fn DoFunc) error {
 
 	// Spawn build function
 	eg.Go(func() error {
-		return c.buildfn(gctx, state, environment, fn, events)
+		return c.buildfn(gctx, state, env, fn, events)
 	})
 
 	return eg.Wait()
@@ -200,7 +201,7 @@ func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment.
 				llb.WithCustomName("[internal] serializing computed values"),
 			)
 
-		ref, err := s.Solve(ctx, st)
+		ref, err := s.Solve(ctx, st, platforms.DefaultSpec())
 		if err != nil {
 			return nil, err
 		}

cmd/dagger/cmd/compute.go

@@ -9,6 +9,8 @@ import (
 	"strings"
 
 	"cuelang.org/go/cue"
+	"github.com/containerd/containerd/platforms"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"go.dagger.io/dagger/cmd/dagger/cmd/common"
 	"go.dagger.io/dagger/cmd/dagger/logger"
 	"go.dagger.io/dagger/compiler"
@@ -41,8 +43,9 @@ var computeCmd = &cobra.Command{
 		doneCh := common.TrackCommand(ctx, cmd)
 
 		st := &state.State{
-			Name: "FIXME",
+			Name:         "FIXME",
-			Path: args[0],
+			Architecture: platforms.Format(specs.Platform{OS: "linux", Architecture: "amd64"}),
+			Path:         args[0],
 			Plan: state.Plan{
 				Module: args[0],
 			},

cmd/dagger/cmd/edit.go

@@ -73,6 +73,7 @@ var editCmd = &cobra.Command{
 			lg.Fatal().Err(err).Msg("failed to decode file")
 		}
 		st.Name = newState.Name
+		st.Architecture = newState.Architecture
 		st.Plan = newState.Plan
 		st.Inputs = newState.Inputs
 

cmd/dagger/cmd/new.go

@@ -34,7 +34,7 @@ var newCmd = &cobra.Command{
 
 		st, err := project.Create(ctx, name, state.Plan{
 			Package: viper.GetString("package"),
-		})
+		}, viper.GetString("architecture"))
 
 		if err != nil {
 			lg.Fatal().Err(err).Msg("failed to create environment")
@@ -46,6 +46,7 @@ var newCmd = &cobra.Command{
 
 func init() {
 	newCmd.Flags().StringP("package", "p", "", "references the name of the Cue package within the module to use as a plan. Default: defer to cue loader")
+	newCmd.Flags().StringP("architecture", "a", "", "architecture of the running pipeline. Default: host architecture")
 	if err := viper.BindPFlags(newCmd.Flags()); err != nil {
 		panic(err)
 	}

environment/environment.go

@@ -7,6 +7,8 @@ import (
 
 	"cuelang.org/go/cue"
 	cueflow "cuelang.org/go/tools/flow"
+	"github.com/containerd/containerd/platforms"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"go.dagger.io/dagger/compiler"
 	"go.dagger.io/dagger/solver"
 	"go.dagger.io/dagger/state"
@@ -137,7 +139,7 @@ func (e *Environment) Up(ctx context.Context, s solver.Solver) error {
 	flow := cueflow.New(
 		&cueflow.Config{},
 		e.src.Cue(),
-		newTaskFunc(newPipelineRunner(e.computed, s)),
+		newTaskFunc(newPipelineRunner(e.computed, s, e.state.Architecture)),
 	)
 	if err := flow.Run(ctx); err != nil {
 		return err
@@ -176,7 +178,7 @@ func noOpRunner(t *cueflow.Task) error {
 	return nil
 }
 
-func newPipelineRunner(computed *compiler.Value, s solver.Solver) cueflow.RunnerFunc {
+func newPipelineRunner(computed *compiler.Value, s solver.Solver, platform string) cueflow.RunnerFunc {
 	return cueflow.RunnerFunc(func(t *cueflow.Task) error {
 		ctx := t.Context()
 		lg := log.
@@ -197,7 +199,24 @@ func newPipelineRunner(computed *compiler.Value, s solver.Solver) cueflow.Runner
 				Msg("dependency detected")
 		}
 		v := compiler.Wrap(t.Value())
-		p := NewPipeline(v, s)
+
+		var pipelinePlatform specs.Platform
+		if platform == "" {
+			pipelinePlatform = specs.Platform{OS: "linux", Architecture: "amd64"}
+		} else {
+			p, err := platforms.Parse(platform)
+			if err != nil {
+				// Record the error
+				span.AddEvent("command", trace.WithAttributes(
+					attribute.String("error", err.Error()),
+				))
+
+				return err
+			}
+			pipelinePlatform = p
+		}
+
+		p := NewPipeline(v, s, pipelinePlatform)
 		err := p.Run(ctx)
 		if err != nil {
 			// Record the error

environment/pipeline.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"cuelang.org/go/cue"
+	bkplatforms "github.com/containerd/containerd/platforms"
 	"github.com/docker/distribution/reference"
 	bk "github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/client/llb"
@@ -22,6 +23,7 @@ import (
 	bkgw "github.com/moby/buildkit/frontend/gateway/client"
 	bkpb "github.com/moby/buildkit/solver/pb"
 	digest "github.com/opencontainers/go-digest"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/rs/zerolog/log"
 	"gopkg.in/yaml.v3"
 
@@ -44,15 +46,17 @@ type Pipeline struct {
 	name     string
 	s        solver.Solver
 	state    llb.State
+	platform specs.Platform // Architecture constraint
 	result   bkgw.Reference
 	image    dockerfile2llb.Image
 	computed *compiler.Value
 }
 
-func NewPipeline(code *compiler.Value, s solver.Solver) *Pipeline {
+func NewPipeline(code *compiler.Value, s solver.Solver, platform specs.Platform) *Pipeline {
 	return &Pipeline{
 		code:     code,
 		name:     code.Path().String(),
+		platform: platform,
 		s:        s,
 		state:    llb.Scratch(),
 		computed: compiler.NewValue(),
@@ -229,7 +233,7 @@ func (p *Pipeline) run(ctx context.Context) error {
 		// so that errors map to the correct cue path.
 		// FIXME: might as well change FS to make every operation
 		// synchronous.
-		p.result, err = p.s.Solve(ctx, p.state)
+		p.result, err = p.s.Solve(ctx, p.state, p.platform)
 		if err != nil {
 			return err
 		}
@@ -335,7 +339,7 @@ func (p *Pipeline) Copy(ctx context.Context, op *compiler.Value, st llb.State) (
 		return st, err
 	}
 	// Execute 'from' in a tmp pipeline, and use the resulting fs
-	from := NewPipeline(op.Lookup("from"), p.s)
+	from := NewPipeline(op.Lookup("from"), p.s, p.platform)
 	if err := from.Run(ctx); err != nil {
 		return st, err
 	}
@@ -591,7 +595,7 @@ func (p *Pipeline) mount(ctx context.Context, dest string, mnt *compiler.Value)
 		return nil, fmt.Errorf("invalid mount: should have %s structure",
 			"{from: _, path: string | *\"/\"}")
 	}
-	from := NewPipeline(mnt.Lookup("from"), p.s)
+	from := NewPipeline(mnt.Lookup("from"), p.s, p.platform)
 	if err := from.Run(ctx); err != nil {
 		return nil, err
 	}
@@ -737,7 +741,7 @@ func parseStringOrSecret(ctx context.Context, ss solver.SecretsStore, v *compile
 
 func (p *Pipeline) Load(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
 	// Execute 'from' in a tmp pipeline, and use the resulting fs
-	from := NewPipeline(op.Lookup("from"), p.s)
+	from := NewPipeline(op.Lookup("from"), p.s, p.platform)
 	if err := from.Run(ctx); err != nil {
 		return st, err
 	}
@@ -795,7 +799,8 @@ func (p *Pipeline) FetchContainer(ctx context.Context, op *compiler.Value, st ll
 
 	// Load image metadata and convert to to LLB.
 	p.image, err = p.s.ResolveImageConfig(ctx, ref.String(), llb.ResolveImageConfigOpt{
-		LogName: p.vertexNamef("load metadata for %s", ref.String()),
+		LogName:  p.vertexNamef("load metadata for %s", ref.String()),
+		Platform: &p.platform,
 	})
 	if err != nil {
 		return st, err
@@ -855,18 +860,18 @@ func (p *Pipeline) PushContainer(ctx context.Context, op *compiler.Value, st llb
 		return st, err
 	}
 
-	if digest, ok := resp.ExporterResponse["containerimage.digest"]; ok {
+	if dgst, ok := resp.ExporterResponse["containerimage.digest"]; ok {
 		imageRef := fmt.Sprintf(
 			"%s@%s",
 			resp.ExporterResponse["image.name"],
-			digest,
+			dgst,
 		)
 
 		return st.File(
 			llb.Mkdir("/dagger", fs.FileMode(0755)),
 			llb.WithCustomName(p.vertexNamef("Mkdir /dagger")),
 		).File(
-			llb.Mkfile("/dagger/image_digest", fs.FileMode(0644), []byte(digest)),
+			llb.Mkfile("/dagger/image_digest", fs.FileMode(0644), []byte(dgst)),
 			llb.WithCustomName(p.vertexNamef("Storing image digest to /dagger/image_digest")),
 		).File(
 			llb.Mkfile("/dagger/image_ref", fs.FileMode(0644), []byte(imageRef)),
@@ -1068,7 +1073,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S
 	// docker build context. This can come from another component, so we need to
 	// compute it first.
 	if dockerContext.Exists() {
-		from := NewPipeline(op.Lookup("context"), p.s)
+		from := NewPipeline(op.Lookup("context"), p.s, p.platform)
 		if err := from.Run(ctx); err != nil {
 			return st, err
 		}
@@ -1107,6 +1112,11 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S
 		opts["no-cache"] = ""
 	}
 
+	// Set platform to configured one if no one is defined
+	if opts["platform"] == "" {
+		opts["platform"] = bkplatforms.Format(p.platform)
+	}
+
 	req := bkgw.SolveRequest{
 		Frontend:    "dockerfile.v0",
 		FrontendOpt: opts,

go.mod

@@ -7,6 +7,7 @@ require (
 	filippo.io/age v1.0.0
 	github.com/KromDaniel/jonson v0.0.0-20180630143114-d2f9c3c389db
 	github.com/containerd/console v1.0.3
+	github.com/containerd/containerd v1.5.4 // indirect
 	github.com/docker/buildx v0.6.2
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/emicklei/proto v1.9.0 // indirect
@@ -21,6 +22,7 @@ require (
 	github.com/moby/buildkit v0.9.1
 	github.com/morikuni/aec v1.0.0
 	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/rs/zerolog v1.23.0
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/viper v1.8.1

solver/solver.go

@@ -16,6 +16,7 @@ import (
 	"github.com/moby/buildkit/session"
 	bkpb "github.com/moby/buildkit/solver/pb"
 	"github.com/opencontainers/go-digest"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/rs/zerolog/log"
 )
 
@@ -79,9 +80,9 @@ func (s Solver) AddCredentials(target, username, secret string) {
 	s.opts.Auth.AddCredentials(target, username, secret)
 }
 
-func (s Solver) Marshal(ctx context.Context, st llb.State) (*bkpb.Definition, error) {
+func (s Solver) Marshal(ctx context.Context, st llb.State, co ...llb.ConstraintsOpt) (*bkpb.Definition, error) {
 	// FIXME: do not hardcode the platform
-	def, err := st.Marshal(ctx, llb.LinuxAmd64)
+	def, err := st.Marshal(ctx, co...)
 	if err != nil {
 		return nil, err
 	}
@@ -126,9 +127,9 @@ func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw.
 }
 
 // Solve will block until the state is solved and returns a Reference.
-func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) {
+// It takes a platform as argument which correspond to the architecture.
-	// marshal llb
+func (s Solver) Solve(ctx context.Context, st llb.State, platform specs.Platform) (bkgw.Reference, error) {
-	def, err := s.Marshal(ctx, st)
+	def, err := s.Marshal(ctx, st, llb.Platform(platform))
 	if err != nil {
 		return nil, err
 	}

state/project.go

@@ -235,7 +235,7 @@ func (w *Project) Save(ctx context.Context, st *State) error {
 	return nil
 }
 
-func (w *Project) Create(ctx context.Context, name string, plan Plan) (*State, error) {
+func (w *Project) Create(ctx context.Context, name string, plan Plan, arch string) (*State, error) {
 	if _, err := w.Get(ctx, name); err == nil {
 		return nil, ErrExist
 	}
@@ -263,7 +263,8 @@ func (w *Project) Create(ctx context.Context, name string, plan Plan) (*State, e
 		Plan: Plan{
 			Package: pkg,
 		},
-		Name: name,
+		Name:         name,
+		Architecture: arch,
 	}
 
 	data, err := yaml.Marshal(st)

state/project_test.go

@@ -32,9 +32,10 @@ func TestProject(t *testing.T) {
 	// Create
 	st, err := project.Create(ctx, "test", Plan{
 		Module: ".",
-	})
+	}, "linux/amd64")
 	require.NoError(t, err)
 	require.Equal(t, "test", st.Name)
+	require.Equal(t, "linux/amd64", st.Architecture)
 
 	// Open
 	project, err = Open(ctx, root)
@@ -51,6 +52,7 @@ func TestProject(t *testing.T) {
 	env, err := project.Get(ctx, "test")
 	require.NoError(t, err)
 	require.Equal(t, "test", env.Name)
+	require.Equal(t, "linux/amd64", env.Architecture)
 
 	// Save
 	require.NoError(t, env.SetInput("foo", TextInput("bar")))
@@ -82,7 +84,7 @@ func TestEncryption(t *testing.T) {
 
 	_, err = project.Create(ctx, "test", Plan{
 		Module: ".",
-	})
+	}, "linux/amd64")
 	require.NoError(t, err)
 
 	// Set a plaintext input, make sure it is not encrypted

state/state.go

@@ -24,6 +24,9 @@ type State struct {
 	// FIXME: store multiple names?
 	Name string `yaml:"name,omitempty"`
 
+	// Architecture execution
+	Architecture string `yaml:"architecture,omitempty"`
+
 	// User Inputs
 	Inputs map[string]Input `yaml:"inputs,omitempty"`
 

tests/core.bats

@@ -159,6 +159,28 @@ setup() {
     "$DAGGER" up
 }
 
+@test "core: arch config" {
+  dagger init
+
+  # Test for amd64 architecture
+  dagger_new_with_plan test-amd "$TESTDIR"/core/arch-config "linux/amd64"
+
+  # Set arch expected value
+  "$DAGGER" -e test-amd input text targetArch "x86_64"
+
+  # Up amd
+  "$DAGGER" -e test-amd up
+
+  # Test for amd64 architecture
+  dagger_new_with_plan test-arm "$TESTDIR"/core/arch-config "linux/arm64"
+
+  # Set arch expected value
+  "$DAGGER" -e test-arm input text targetArch "aarch64"
+
+  # Up arm
+  "$DAGGER" -e test-arm up
+}
+
 @test "compute: exclude" {
     "$DAGGER" up --project "$TESTDIR"/compute/exclude
 }

tests/core/arch-config/arch-config.cue

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"alpha.dagger.io/dagger/op"
+	"alpha.dagger.io/dagger"
+)
+
+targetArch: dagger.#Input & {string}
+
+TestFetch: #up: [
+	op.#FetchContainer & {
+		ref: "docker.io/alpine"
+	},
+
+	op.#Exec & {
+		args: ["/bin/sh", "-c", "echo $(uname -a) >> /arch.txt"]
+		always: true
+	},
+
+	op.#Exec & {
+		args: ["/bin/sh", "-c", """
+			cat /arch.txt | grep "$TARGET_ARCH"
+			"""]
+		env: TARGET_ARCH: targetArch
+	},
+]
+
+TestBuild: #up: [
+	op.#DockerBuild & {
+		dockerfile: """
+				FROM alpine
+				
+				RUN echo $(uname -a) > /arch.txt
+			"""
+	},
+
+	op.#Exec & {
+		args: ["/bin/sh", "-c", """
+			cat /arch.txt | grep "$TARGET_ARCH"
+			"""]
+		env: TARGET_ARCH: targetArch
+	},
+]
+
+TestLoad: #up: [
+	op.#Load & {
+		from: TestBuild
+	},
+
+	// Compare arch
+	op.#Exec & {
+		args: ["/bin/sh", "-c", "diff /build/arch.txt /fetch/arch.txt"]
+		mount: {
+			"/build": from: TestBuild
+			"/fetch": from: TestFetch
+		}
+	},
+]

tests/helpers.bash

@@ -20,10 +20,19 @@ common_setup() {
 dagger_new_with_plan() {
     local name="$1"
     local sourcePlan="$2"
+    local arch="$3"
 
     cp -a "$sourcePlan"/* "$DAGGER_PROJECT"
 
-    "$DAGGER" new "$name"
+    local opts=""
+    if [ -n "$arch" ];
+    then
+      opts="-a $arch"
+    fi
+
+    # Need word splitting to take in account "-a" and "$arch"
+    # shellcheck disable=SC2086
+    "$DAGGER" new "$name" ${opts}
 }
 
 dagger_new_with_env() {