Merge pull request #1238 from talentedmrjones/europa-reconcile-with-spec

Europa: reconcile with spec

docs/reference/README.md

@@ -19,10 +19,10 @@
 - [dagger/op](./dagger/op.md) - op: low-level operations for Dagger processing pipelines
 - [docker](./docker/README.md) - Docker container operations
 - [docker/compose](./docker/compose.md) - Docker-compose operations
-- [europa/dagger](./europa/dagger/README.md) - The Dagger API.
+- [europa/dagger](./europa/dagger/README.md) - -
 - [europa/dagger/engine](./europa/dagger/engine/README.md) - -
 - [europa/dagger/engine/spec](./europa/dagger/engine/spec/README.md) - Placeholder package, to keep docs generating tool happy.
-- [europa/dagger/engine/spec/engine](./europa/dagger/engine/spec/engine.md) - HTTP operations
+- [europa/dagger/engine/spec/engine](./europa/dagger/engine/spec/engine.md) - The Dagger API.
 - [gcp](./gcp/README.md) - Google Cloud Platform
 - [gcp/cloudrun](./gcp/cloudrun.md) - -
 - [gcp/gcr](./gcp/gcr.md) - Google Container Registry

docs/reference/europa/dagger/README.md

@@ -4,8 +4,6 @@ sidebar_label: dagger
 
 # alpha.dagger.io/europa/dagger
 
-The Dagger API.
-
 ```cue
 import "alpha.dagger.io/europa/dagger"
 ```
@@ -72,7 +70,7 @@ _No output._
 
 ## dagger.#Stream
 
-A reference to a stream of bytes, for example:  - The standard output or error stream of a command  - The standard input stream of a command  - The contents of a file or named pipe
+A stream of bytes
 
 ### dagger.#Stream Inputs
 

docs/reference/europa/dagger/engine/README.md

@@ -20,16 +20,6 @@ _No input._
 
 _No output._
 
-## engine.#Context
-
-### engine.#Context Inputs
-
-_No input._
-
-### engine.#Context Outputs
-
-_No output._
-
 ## engine.#Exec
 
 Execute a command in a container

docs/reference/europa/dagger/engine/spec/engine.md

@@ -4,7 +4,7 @@ sidebar_label: engine
 
 # alpha.dagger.io/europa/dagger/engine/spec/engine
 
-HTTP operations
+The Dagger API.
 
 ```cue
 import "alpha.dagger.io/europa/dagger/engine/spec/engine"
@@ -34,16 +34,6 @@ _No input._
 
 _No output._
 
-## engine.#Context
-
-### engine.#Context Inputs
-
-_No input._
-
-### engine.#Context Outputs
-
-_No output._
-
 ## engine.#Copy
 
 ### engine.#Copy Inputs
@@ -64,6 +54,18 @@ _No input._
 
 _No output._
 
+## engine.#DAG
+
+A special kind of program which `dagger` can execute.
+
+### engine.#DAG Inputs
+
+_No input._
+
+### engine.#DAG Outputs
+
+_No output._
+
 ## engine.#Exec
 
 Execute a command in a container

plan/plan.go

@@ -59,7 +59,7 @@ func (p *Plan) Source() *compiler.Value {
 // registerLocalDirectories scans the context for local imports.
 // BuildKit requires to known the list of directories ahead of time.
 func (p *Plan) registerLocalDirs() error {
-	imports, err := p.source.Lookup("context.imports").Fields()
+	imports, err := p.source.Lookup("inputs.directories").Fields()
 	if err != nil {
 		return err
 	}

plan/task/localDirectory.go

@@ -4,19 +4,20 @@ import (
 	"context"
 
 	"github.com/moby/buildkit/client/llb"
+	"github.com/rs/zerolog/log"
 	"go.dagger.io/dagger/compiler"
 	"go.dagger.io/dagger/plancontext"
 	"go.dagger.io/dagger/solver"
 )
 
 func init() {
-	Register("Import", func() Task { return &importTask{} })
+	Register("LocalDirectory", func() Task { return &localDirectoryTask{} })
 }
 
-type importTask struct {
+type localDirectoryTask struct {
 }
 
-func (c importTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+func (c localDirectoryTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
 	var dir struct {
 		Path    string
 		Include []string
@@ -27,6 +28,8 @@ func (c importTask) Run(ctx context.Context, pctx *plancontext.Context, s solver
 		return nil, err
 	}
 
+	lg := log.Ctx(ctx)
+	lg.Debug().Str("path", dir.Path).Msg("loading local directory")
 	opts := []llb.LocalOption{
 		withCustomName(v, "Local %s", dir.Path),
 		// Without hint, multiple `llb.Local` operations on the
@@ -69,6 +72,6 @@ func (c importTask) Run(ctx context.Context, pctx *plancontext.Context, s solver
 
 	fs := pctx.FS.New(result)
 	return compiler.NewValue().FillFields(map[string]interface{}{
-		"fs": fs.MarshalCUE(),
+		"contents": fs.MarshalCUE(),
 	})
 }

stdlib/europa/dagger/engine/plan.cue

@@ -2,50 +2,71 @@ package engine
 
 // A deployment plan executed by `dagger up`
 #Plan: {
-	context: #Context
+	// Receive inputs from the client
-	actions: [string]: _
+	inputs: {
-}
+		// Receive directories
-
+		directories: [string]: _#inputDirectory
-// FIXME: Platform spec here
+		// Securely receive secrets
-#Platform: string
+		secrets: [string]: _#inputSecret
-
-#Context: {
-	// Platform to target
-	platform?: #Platform
-
-	// Import directories
-	imports: [string]: {
-		_type: "Import"
-
-		path: string
-		include?: [...string]
-		exclude?: [...string]
-		fs: #FS
 	}
 
-	// Securely load external secrets
+	// Forward network services to and from the client
-	secrets: [string]: {
+	proxy: [string]: _#proxyEndpoint
-		// Secrets can be securely mounted into action containers as a file
-		contents: #Secret
 
-		{
+	// Execute actions in containers
-			_type: "SecretFile"
+	actions: {
-			// Read secret from a file
+		...
-			path: string
+	}
-		} | {
+}
-			_type: "SecretEnv"
+
-			// Read secret from an environment variable ON THE CLIENT MACHINE
+_#inputDirectory: {
-			envvar: string
+	// Import from this path ON THE CLIENT MACHINE
-		}
+	// Example: "/Users/Alice/dev/todoapp/src"
-	}
+	_type: "LocalDirectory"
-
+	path:  string
-	services: [string]: {
+
-		service: #Service
+	// Filename patterns to include
-		_type:   "Service"
+	// Example: ["*.go", "Dockerfile"]
-		{
+	include?: [...string]
-			unix: string
+
-		} | {
+	// Filename patterns to exclude
-			npipe: string
+	// Example: ["node_modules"]
-		}
+	exclude?: [...string]
+
+	// Imported filesystem contents
+	// Use this as input for actions requiring an #FS field
+	contents: #FS
+}
+
+// Securely receive a secret from the client
+_#inputSecret: {
+	// Reference to the secret contents
+	// Use this by securely mounting it into a container.
+	// See universe.dagger.io/docker.#Run.mounts
+	// FIXME: `contents` field name causes confusion (not actually the secret contents..)
+	contents: #Secret
+
+	{
+		// Read secret from a file ON THE CLIENT MACHINE
+		_type: "SecretFile"
+		path:  string
+	} | {
+		// Read secret from an environment variable ON THE CLIENT MACHINE
+		_type:  "SecretEnv"
+		envvar: string
+	}
+}
+
+// Forward a network endpoint to and from the client
+_#proxyEndpoint: {
+	// Service endpoint can be proxied to action containers as unix sockets
+	// FIXME: should #Service be renamed to #ServiceEndpoint or #Endpoint? Naming things is hard...
+	// FIXME: reconcile with spec
+	_type:   "Service"
+	service: #Service
+	{
+		unix: string
+	} | {
+		npipe: string
 	}
 }

stdlib/europa/dagger/engine/spec/engine/address.cue

@@ -0,0 +1,4 @@
+package engine
+
+// A network service address
+#Address: string & =~"^(tcp://|unix://|udp://).*"

stdlib/europa/dagger/engine/spec/engine/http.cue

@@ -1,6 +1,7 @@
-// HTTP operations
 package engine
 
+// HTTP operations
+
 // Raw buildkit API
 //
 // package llb // import "github.com/moby/buildkit/client/llb"

stdlib/europa/dagger/engine/spec/engine/plan.cue

@@ -0,0 +1,102 @@
+// The Dagger API.
+package engine
+
+// A deployment plan executed by `dagger up`
+#Plan: #DAG
+
+// A special kind of program which `dagger` can execute.
+#DAG: {
+	// Receive inputs from the client
+	inputs: {
+		// Receive directories
+		directories: [name=string]: _#inputDirectory
+		// Securely receive secrets
+		secrets: [name=string]: _#inputSecret
+		// Receive runtime parameters
+		params: [name=string]: _
+	}
+
+	// Send outputs to the client
+	outputs: {
+		directories: [name=string]: _#outputDirectory
+	}
+
+	// Forward network services to and from the client
+	proxy: [name=string]: _#proxyEndpoint
+
+	// Execute actions in containers
+	actions: {
+		...
+	}
+}
+
+_#inputDirectory: {
+	// Import from this path ON THE CLIENT MACHINE
+	// Example: "/Users/Alice/dev/todoapp/src"
+	source: string
+
+	// Filename patterns to include
+	// Example: ["*.go", "Dockerfile"]
+	include?: [...string]
+
+	// Filename patterns to exclude
+	// Example: ["node_modules"]
+	exclude?: [...string]
+
+	// Imported filesystem contents
+	// Use this as input for actions requiring an #FS field
+	contents: #FS
+}
+
+_#outputDirectory: {
+	// Filesystem contents to export
+	// Reference an #FS field produced by an action
+	contents: #FS
+
+	// Export to this path ON THE CLIENT MACHINE
+	dest: string
+}
+
+// Securely receive a secret from the client
+_#inputSecret: {
+	// Reference to the secret contents
+	// Use this by securely mounting it into a container.
+	// See universe.dagger.io/docker.#Run.mounts
+	// FIXME: `contents` field name causes confusion (not actually the secret contents..)
+	contents: #Secret
+
+	{
+		// Execute a command ON THE CLIENT MACHINE and read secret from standard output
+		command: [string, ...string] | string
+		// Execute command in an interactive terminal
+		//  for example to prompt for a passphrase
+		interactive: true | *false
+	} | {
+		// Read secret from a file ON THE CLIENT MACHINE
+		path: string
+	} | {
+		// Read secret from an environment variable ON THE CLIENT MACHINE
+		envvar: string
+	}
+}
+
+// Forward a network endpoint to and from the client
+_#proxyEndpoint: {
+	// Service endpoint can be proxied to action containers as unix sockets
+	// FIXME: should #Service be renamed to #ServiceEndpoint or #Endpoint? Naming things is hard...
+	endpoint: #Service
+
+	{
+		// Listen for connections ON THE CLIENT MACHINE, proxy to actions
+		listen: #Address
+	} | {
+		// Connect to a remote endpoint FROM THE CLIENT MACHINE, proxy to actions
+		connect: #Address
+	} | {
+		// Proxy to/from the contents of a file ON THE CLIENT MACHINE
+		filepath: string
+	} | {
+		// Proxy to/from standard input and output of a command ON THE CLIENT MACHINE
+		command: [string, ...string] | string
+	}
+}

stdlib/europa/dagger/plan.cue

@@ -1,102 +1,11 @@
-// The Dagger API.
 package dagger
 
+import (
+	"alpha.dagger.io/europa/dagger/engine/spec/engine"
+)
+
 // A deployment plan executed by `dagger up`
-#Plan: #DAG
+#Plan: engine.#Plan
 
 // A special kind of program which `dagger` can execute.
-#DAG: {
+#DAG: engine.#DAG
-	// Receive inputs from the client
-	input: {
-		// Receive directories
-		directories: [name=string]: _#inputDirectory
-		// Securely receive secrets
-		secrets: [name=string]: _#inputSecret
-		// Receive runtime parameters
-		params: [name=string]: _
-	}
-
-	// Send outputs to the client
-	output: {
-		directories: [name=string]: _#outputDirectory
-	}
-
-	// Forward network services to and from the client
-	proxy: [name=string]: _#proxyEndpoint
-
-	// Execute actions in containers
-	actions: {
-		...
-	}
-}
-
-_#inputDirectory: {
-	// Import from this path ON THE CLIENT MACHINE
-	// Example: "/Users/Alice/dev/todoapp/src"
-	source: string
-
-	// Filename patterns to include
-	// Example: ["*.go", "Dockerfile"]
-	include?: [...string]
-
-	// Filename patterns to exclude
-	// Example: ["node_modules"]
-	exclude?: [...string]
-
-	// Imported filesystem contents
-	// Use this as input for actions requiring an #FS field
-	contents: #FS
-}
-
-_#outputDirectory: {
-	// Filesystem contents to export
-	// Reference an #FS field produced by an action
-	contents: #FS
-
-	// Export to this path ON THE CLIENT MACHINE
-	dest: string
-}
-
-// Securely receive a secret from the client
-_#inputSecret: {
-	// Reference to the secret contents
-	// Use this by securely mounting it into a container.
-	// See universe.dagger.io/docker.#Run.mounts
-	// FIXME: `contents` field name causes confusion (not actually the secret contents..)
-	contents: #Secret
-
-	{
-		// Execute a command ON THE CLIENT MACHINE and read secret from standard output
-		command: [string, ...string] | string
-		// Execute command in an interactive terminal
-		//  for example to prompt for a passphrase
-		interactive: true | *false
-	} | {
-		// Read secret from a file ON THE CLIENT MACHINE
-		path: string
-	} | {
-		// Read secret from an environment variable ON THE CLIENT MACHINE
-		envvar: string
-	}
-}
-
-// Forward a network endpoint to and from the client
-_#proxyEndpoint: {
-	// Service endpoint can be proxied to action containers as unix sockets
-	// FIXME: should #Service be renamed to #ServiceEndpoint or #Endpoint? Naming things is hard...
-	endpoint: #Service
-
-	{
-		// Listen for connections ON THE CLIENT MACHINE, proxy to actions
-		listen: #Address
-	} | {
-		// Connect to a remote endpoint FROM THE CLIENT MACHINE, proxy to actions
-		connect: #Address
-	} | {
-		// Proxy to/from the contents of a file ON THE CLIENT MACHINE
-		filepath: string
-	} | {
-		// Proxy to/from standard input and output of a command ON THE CLIENT MACHINE
-		command: [string, ...string] | string
-	}
-}

stdlib/europa/dagger/types.cue

@@ -20,17 +20,14 @@ import (
 // by a special filesystem mount designed to minimize leak risk.
 #Secret: engine.#Secret
 
-// A reference to a stream of bytes, for example:
-//  - The standard output or error stream of a command
-//  - The standard input stream of a command
-//  - The contents of a file or named pipe
-#Stream: engine.#Stream
-
 // A reference to a network service endpoint, for example:
 //  - A TCP or UDP port
 //  - A unix socket
 //  - An HTTPS endpoint
 #Service: engine.#Service
 
+// A stream of bytes
+#Stream: engine.#Stream
+
 // A network service address
-#Address: string & =~"^(tcp://|unix://|udp://).*"
+#Address: engine.#Address

tests/plan.bats

@@ -10,31 +10,43 @@ setup() {
   "$DAGGER" --europa up ./plan/hello-europa
 }
 
-@test "plan/context/services invalid schema" {
+@test "plan/proxy invalid schema" {
   cd "$TESTDIR"
-  run "$DAGGER" --europa up ./plan/context/services/invalid_schema.cue
+  run "$DAGGER" --europa up ./plan/proxy/invalid_schema.cue
   assert_failure
 }
 
-@test "plan/context/services invalid value" {
+@test "plan/proxy invalid value" {
   cd "$TESTDIR"
-  run "$DAGGER" --europa up ./plan/context/services/invalid_value.cue
+  run "$DAGGER" --europa up ./plan/proxy/invalid_value.cue
   assert_failure
 }
 
-@test "plan/context/services incomplete unix" {
+@test "plan/proxy incomplete unix" {
   cd "$TESTDIR"
-  run "$DAGGER" --europa up ./plan/context/services/incomplete_unix.cue
+  run "$DAGGER" --europa up ./plan/proxy/incomplete_unix.cue
   assert_failure
 }
 
-@test "plan/context/services incomplete service" {
+@test "plan/proxy incomplete service" {
   cd "$TESTDIR"
-  run "$DAGGER" --europa up ./plan/context/services/incomplete_service.cue
+  run "$DAGGER" --europa up ./plan/proxy/incomplete_service.cue
   assert_output --partial "pipeline was partially executed because of missing inputs"
 }
 
-@test "plan/context/services unix" {
+@test "plan/proxy unix" {
   cd "$TESTDIR"
-  "$DAGGER" --europa up ./plan/context/services/unix.cue
+  "$DAGGER" --europa up ./plan/proxy/unix.cue
+}
+
+@test "plan/inputs/directories exists" {
+  cd "$TESTDIR"
+  "$DAGGER" --europa up ./plan/inputs/directories/exists.cue
+}
+
+@test "plan/inputs/directories conflicting values" {
+  cd "$TESTDIR"
+  run "$DAGGER" --europa up ./plan/inputs/directories/conflicting_values.cue
+	assert_failure
+	assert_output --partial 'failed to up environment: actions.verify.contents: conflicting values "local directory" and "local dfsadf"'
 }

tests/plan/inputs/directories/conflicting_values.cue

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	inputs: directories: test: path: "./plan/inputs/directories"
+	actions: verify: engine.#ReadFile & {
+		input: inputs.directories.test.contents
+		path:  "test.txt"
+	} & {
+		contents: "local dfsadf" // should fail with conflicting values
+	}
+}

tests/plan/inputs/directories/exists.cue

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	inputs: directories: test: path: "./plan/inputs/directories"
+	actions: verify: engine.#ReadFile & {
+		input: inputs.directories.test.contents
+		path:  "test.txt"
+	} & {
+		contents: "local directory"
+	}
+}

tests/plan/inputs/directories/test.txt

@@ -0,0 +1 @@
+local directory

tests/plan/proxy/incomplete_service.cue

@@ -7,8 +7,8 @@ import (
 )
 
 engine.#Plan & {
-	// should fail
+	// should fail due to incomplete service
-	context: services: dockerSocket: {}
+	proxy: dockerSocket: {}
 
 	actions: test: #up: [
 		op.#Load & {
@@ -19,7 +19,7 @@ engine.#Plan & {
 
 		op.#Exec & {
 			always: true
-			mount: "/var/run/docker.sock": stream: context.services.dockerSocket.service
+			mount: "/var/run/docker.sock": stream: proxy.dockerSocket.service
 			args: ["docker", "info"]
 		},
 	]

tests/plan/proxy/incomplete_unix.cue

@@ -7,8 +7,8 @@ import (
 )
 
 engine.#Plan & {
-	// should succeed
+	// should fail because incomplete value
-	context: services: dockerSocket: unix: "/var/run/docker.sock"
+	proxy: dockerSocket: unix: string
 
 	actions: test: #up: [
 		op.#Load & {
@@ -19,7 +19,7 @@ engine.#Plan & {
 
 		op.#Exec & {
 			always: true
-			mount: "/var/run/docker.sock": stream: context.services.dockerSocket.service
+			mount: "/var/run/docker.sock": stream: proxy.dockerSocket.service
 			args: ["docker", "info"]
 		},
 	]

tests/plan/proxy/invalid_schema.cue

@@ -8,7 +8,7 @@ import (
 
 engine.#Plan & {
 	// should fail because of misspelled key
-	context: services: dockerSocket: unx: "/var/run/docker.soc"
+	proxy: dockerSocket: unx: "/var/run/docker.sock"
 
 	actions: test: #up: [
 		op.#Load & {
@@ -19,7 +19,7 @@ engine.#Plan & {
 
 		op.#Exec & {
 			always: true
-			mount: "/var/run/docker.sock": stream: context.services.dockerSocket.service
+			mount: "/var/run/docker.sock": stream: proxy.dockerSocket.service
 			args: ["docker", "info"]
 		},
 	]

tests/plan/proxy/invalid_value.cue

@@ -8,7 +8,7 @@ import (
 
 engine.#Plan & {
 	// should fail because of misspelled value
-	context: services: dockerSocket: unix: "/var/run/docker.soc"
+	proxy: dockerSocket: unix: "/var/run/docker.soc"
 
 	actions: test: #up: [
 		op.#Load & {
@@ -19,7 +19,7 @@ engine.#Plan & {
 
 		op.#Exec & {
 			always: true
-			mount: "/var/run/docker.sock": stream: context.services.dockerSocket.service
+			mount: "/var/run/docker.sock": stream: proxy.dockerSocket.service
 			args: ["docker", "info"]
 		},
 	]

tests/plan/proxy/unix.cue

@@ -8,7 +8,7 @@ import (
 
 engine.#Plan & {
 	// should succeed
-	context: services: dockerSocket: unix: string
+	proxy: dockerSocket: unix: "/var/run/docker.sock"
 
 	actions: test: #up: [
 		op.#Load & {
@@ -19,7 +19,7 @@ engine.#Plan & {
 
 		op.#Exec & {
 			always: true
-			mount: "/var/run/docker.sock": stream: context.services.dockerSocket.service
+			mount: "/var/run/docker.sock": stream: proxy.dockerSocket.service
 			args: ["docker", "info"]
 		},
 	]

tests/tasks/exec/mount_secret.cue

@@ -5,7 +5,7 @@ import (
 )
 
 engine.#Plan & {
-	context: secrets: testSecret: envvar: "TESTSECRET"
+	inputs: secrets: testSecret: envvar: "TESTSECRET"
 	actions: {
 		image: engine.#Pull & {
 			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
@@ -15,7 +15,7 @@ engine.#Plan & {
 			input: image.output
 			mounts: secret: {
 				dest:     "/run/secrets/test"
-				contents: context.secrets.testSecret.contents
+				contents: inputs.secrets.testSecret.contents
 			}
 			args: [
 				"sh", "-c",
@@ -30,7 +30,7 @@ engine.#Plan & {
 			input: image.output
 			mounts: secret: {
 				dest:     "/run/secrets/test"
-				contents: context.secrets.testSecret.contents
+				contents: inputs.secrets.testSecret.contents
 				uid:      42
 				gid:      24
 				mask:     0o666

tests/tasks/exec/mount_service.cue

@@ -5,7 +5,7 @@ import (
 )
 
 engine.#Plan & {
-	context: services: dockerSocket: unix: "/var/run/docker.sock"
+	proxy: dockerSocket: unix: "/var/run/docker.sock"
 
 	actions: {
 		image: engine.#Pull & {
@@ -21,7 +21,7 @@ engine.#Plan & {
 			input: imageWithDocker.output
 			mounts: docker: {
 				dest:     "/var/run/docker.sock"
-				contents: context.services.dockerSocket.service
+				contents: proxy.dockerSocket.service
 			}
 			args: ["docker", "info"]
 		}

tests/tasks/pull/pull_auth.cue

@@ -5,13 +5,13 @@ import (
 )
 
 engine.#Plan & {
-	context: secrets: dockerHubToken: envvar: "DOCKERHUB_TOKEN"
+	inputs: secrets: dockerHubToken: envvar: "DOCKERHUB_TOKEN"
 	actions: pull: engine.#Pull & {
 		source: "daggerio/ci-test:private-pull@sha256:c74f1b1166784193ea6c8f9440263b9be6cae07dfe35e32a5df7a31358ac2060"
 		auth: [{
 			target:   "daggerio/ci-test:private-pull"
 			username: "daggertest"
-			secret:   context.secrets.dockerHubToken.contents
+			secret:   inputs.secrets.dockerHubToken.contents
 		}]
 	} & {
 		// assert result