Europa: integrate core packages, separate universe

Signed-off-by: Solomon Hykes <solomon@dagger.io>

stdlib/dagger/dagger.cue

@@ -3,7 +3,7 @@ package dagger
 
 import (
 	"alpha.dagger.io/dagger/op"
-	"alpha.dagger.io/dagger/engine"
+	"alpha.dagger.io/europa/dagger/engine"
 )
 
 // An artifact such as source code checkout, container image, binary archive...

stdlib/europa/README.md

@@ -0,0 +1,59 @@
+# Europa Core packages
+
+## About this directory
+
+`stdlib/europa/` holds the development version of the Core packages for the upcoming [Europa release](https://github.com/dagger/dagger/issues/1088).
+
+Once Europa is released, `stdlib/europa` will become the new `stdlib/`
+
+## What are Dagger core packages?
+
+Dagger core packages are CUE packages released alongside the Dagger engine, to allow developers to access its features.
+
+### Dagger Core API: `dagger.io/dagger`
+
+*Development import path: `alpha.dagger.io/europa/dagger`*
+
+The Dagger Core API defines core types and utilities for programming Dagger:
+
+* `#Plan`: a complete configuration executable by `dagger up`
+* `#FS` to reference filesystem state
+* `#Secret` to (securely) reference external secrets
+* `#Service` to reference network service endpoints
+* `#Stream` to reference byte streams
+
+### Low-level Engine API: `dagger.io/dagger/engine`
+
+* *Development import path (implemented subset): `alpha.dagger.io/europa/dagger/engine`*
+* *Development importa pth (full spec): `alpha.dagger.io/dagger/europa/dagger/engine/spec/engine`*
+
+`engine` is a low-level API for accessing the raw capabilities of the Dagger Engine. Most developers should use the Dagger Core API instead (`dagger.io/dagger`), but experts and framework developers can target the engine API directly for maximum control.
+
+This API prioritizes robustness, consistency, and feature completeness. It does NOT prioritize developer convenience or leveraging Cue for composition.
+
+In Europa, `engine` will deprecate the following implicit API:
+* Low-level operations defined in `alpha.dagger.io/dagger/op`
+* Imperative DSL to assemble Dockerfile-like arrays as Cue arrays
+* Convention to embed pipelines in the Cue lattice with the special nested definition `#up`
+* Convention to reference filesystem state from the Cue lattice with `@dagger(artifact)`
+* Convention to reference external secrets from the Cue lattice with `@dagger(secret)`
+* Convention to reference external network endpoints from the Cue lattive with `@dagger(stream)`
+* Convention that some operations (specifically `op.#Local`) are meant to be generated at runtime rather than authored manually.
+
+## TODO LIST
+
+* #Scratch: replace with null #FS?
+* Resolve registry auth special case (buildkit does not support scoping registry auth)
+* Are there runtime limitations in….
+     * using hidden fields `_foo` as part of the DAG?
+     * using `if` statements as part of the DAG?
+     * using inlined Cue expressions as part of the DAG?
+* Readability of error messages
+  * At a minimum don’t make it worse!
+  * Small improvements are good (eg. 
+* Make sure we don’t make error messages LESS readable
+* [Outstanding questions on proxy features](https://github.com/dagger/dagger/pull/1117#discussion_r765211280)
+* [Outstanding questions on #Stream and emulating unix pipes with them](https://github.com/dagger/dagger/pull/1117#discussion_r766145864)
+* [Outstanding questions on engine.#Pull and information loss](https://github.com/dagger/dagger/pull/1117#discussion_r765219049)
+* [Outstanding questions on global registry auth scope in buildkit](https://github.com/dagger/dagger/pull/1117#discussion_r765963051)
+* [Outstanding questions on platform key](https://github.com/dagger/dagger/pull/1117#discussion_r766085610)

stdlib/europa/dagger/engine/image.cue

@@ -9,7 +9,7 @@ package engine
 //   - "index.docker.io/dagger:latest@sha256:a89cb097693dd354de598d279c304a1c73ee550fbfff6d9ee515568e0c749cfe"
 #Ref: string
 
-// Container image config. See [OCI](https://opencontainers.org/).
+// Container image config. See [OCI](https://www.opencontainers.org/).
 // Spec left open on purpose to account for additional fields.
 // [Image Spec](https://github.com/opencontainers/image-spec/blob/main/specs-go/v1/config.go)
 // [Docker Superset](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/dockerfile2llb/image.go)

stdlib/europa/dagger/engine/spec/engine/exec.cue

@@ -0,0 +1,64 @@
+package engine
+
+// Execute a command in a container
+#Exec: {
+	_exec: {}
+
+	// Container filesystem
+	input: #FS
+
+	// Mounts
+	mounts: [...#Mount]
+
+	// Command to execute
+	args: [...string] | string
+
+	// Environment variables
+	environ: [...string]
+
+	// Working directory
+	workdir?: string
+
+	// Optionally attach to command standard input stream
+	stdin?: #Stream
+
+	// Optionally attach to command standard output stream
+	stdout?: #Stream
+
+	// Optionally attach to command standard error stream
+	stderr?: #Stream
+
+	// Modified filesystem
+	output: #FS
+
+	// Command exit code
+	exit: int
+}
+
+// A transient filesystem mount.
+#Mount: {
+	dest: string
+	{
+		contents: #CacheDir | #TempDir | #Service
+	} | {
+		contents: #FS
+		source:   string | *"/"
+		ro:       true | *false
+	} | {
+		contents: #Secret
+		uid:      uint32 | *0
+		gid:      uint32 | *0
+		optional: true | *false
+	}
+}
+
+// A (best effort) persistent cache dir
+#CacheDir: {
+	id:          string
+	concurrency: *"shared" | "private" | "locked"
+}
+
+// A temporary directory for command execution
+#TempDir: {
+	size?: int64
+}

stdlib/europa/dagger/engine/spec/engine/fs.cue

@@ -0,0 +1,56 @@
+package engine
+
+// A filesystem state
+#FS: {
+	_fs: ID: string
+}
+
+// Produce an empty directory
+// FIXME: replace with a null value for #FS?
+#Scratch: {
+	_scratch: {}
+
+	output: #FS
+}
+
+#ReadFile: {
+	_readFile: {}
+
+	input:    #FS
+	path:     string
+	contents: string
+	output:   #FS
+}
+
+#WriteFile: {
+	_writeFile: {}
+
+	input:    #FS
+	path:     string
+	contents: string
+	output:   #FS
+}
+
+#Copy: {
+	_copy: {}
+
+	input: #FS
+	#CopyInfo
+	output: #FS
+}
+
+#CopyInfo: {
+	source: {
+		root: #FS
+		path: string | *"/"
+	}
+	dest: string
+}
+
+#Merge: {
+	_merge: {}
+
+	input: #FS
+	layers: [...#CopyInfo]
+	output: #FS
+}

stdlib/europa/dagger/engine/spec/engine/git.cue

@@ -0,0 +1,19 @@
+package engine
+
+// Push a directory to a git remote
+#GitPush: {
+	gitPush: {}
+
+	input:  #FS
+	remote: string
+	ref:    string
+}
+
+// Pull a directory from a git remote
+#GitPull: {
+	gitPull: {}
+
+	remote: string
+	ref:    string
+	output: #FS
+}

stdlib/europa/dagger/engine/spec/engine/image.cue

@@ -0,0 +1,88 @@
+package engine
+
+// Container image config
+// See [OCI](https://www.opencontainers.org)
+#ImageConfig: {
+	env?: [...string]
+	user?: string
+	command?: [...string]
+	// FIXME
+}
+
+// Upload a container image to a remote repository
+#Push: {
+	push: {}
+
+	// Target repository address
+	dest: #Ref
+
+	// Filesystem contents to push
+	input: #FS
+
+	// Container image config
+	config: #ImageConfig
+
+	// Authentication
+	auth: [...{
+		target:   string
+		username: string
+		secret:   string | #Secret
+	}]
+
+	// Complete ref of the pushed image, including digest
+	result: #Ref
+}
+
+// Download a container image from a remote repository
+#Pull: {
+	pull: {}
+
+	// Repository source ref
+	source: #Ref
+	// Authentication
+	auth: [...{
+		target:   string
+		username: string
+		secret:   string | #Secret
+	}]
+
+	// Root filesystem of downloaded image
+	output: #FS
+
+	// Complete ref of downloaded image (including digest)
+	result: #Ref
+
+	// Downloaded container image config
+	config: #ImageConfig
+}
+
+// A ref is an address for a remote container image
+//
+// Examples:
+//   - "index.docker.io/dagger"
+//   - "dagger"
+//   - "index.docker.io/dagger:latest"
+//   - "index.docker.io/dagger:latest@sha256:a89cb097693dd354de598d279c304a1c73ee550fbfff6d9ee515568e0c749cfe"
+#Ref: string
+
+// Build a container image using buildkit
+#Build: {
+	build: {}
+
+	// Source directory to build
+	source: #FS
+	{
+		frontend:   "dockerfile"
+		dockerfile: {
+			path: string | *"Dockerfile"
+		} | {
+			contents: string
+		}
+	}
+
+	// Root filesystem produced by build
+	output: #FS
+
+	// Container image config produced by build
+	config: #ImageConfig
+}

stdlib/europa/dagger/engine/spec/engine/secret.cue

@@ -0,0 +1,6 @@
+package engine
+
+// An external secret
+#Secret: {
+	_secret: ID: string
+}

stdlib/europa/dagger/engine/spec/engine/service.cue

@@ -0,0 +1,6 @@
+package engine
+
+// An external network service
+#Service: {
+	_service: ID: string
+}

stdlib/europa/dagger/engine/spec/engine/stream.cue

@@ -0,0 +1,6 @@
+package engine
+
+// A stream of bytes
+#Stream: {
+	_stream: ID: string
+}

stdlib/europa/dagger/engine/spec/spec.cue

@@ -0,0 +1,2 @@
+// Placeholder package, to keep docs generating tool happy.
+package spec

stdlib/europa/dagger/plan.cue

@@ -0,0 +1,102 @@
+// The Dagger API.
+package dagger
+
+// A deployment plan executed by `dagger up`
+#Plan: #DAG
+
+// A special kind of program which `dagger` can execute.
+#DAG: {
+	// Receive inputs from the client
+	input: {
+		// Receive directories
+		directories: [name=string]: _#inputDirectory
+		// Securely receive secrets
+		secrets: [name=string]: _#inputSecret
+		// Receive runtime parameters
+		params: [name=string]: _
+	}
+
+	// Send outputs to the client
+	output: {
+		directories: [name=string]: _#outputDirectory
+	}
+
+	// Forward network services to and from the client
+	proxy: [name=string]: _#proxyEndpoint
+
+	// Execute actions in containers
+	actions: {
+		...
+	}
+}
+
+_#inputDirectory: {
+	// Import from this path ON THE CLIENT MACHINE
+	// Example: "/Users/Alice/dev/todoapp/src"
+	source: string
+
+	// Filename patterns to include
+	// Example: ["*.go", "Dockerfile"]
+	include?: [...string]
+
+	// Filename patterns to exclude
+	// Example: ["node_modules"]
+	exclude?: [...string]
+
+	// Imported filesystem contents
+	// Use this as input for actions requiring an #FS field
+	contents: #FS
+}
+
+_#outputDirectory: {
+	// Filesystem contents to export
+	// Reference an #FS field produced by an action
+	contents: #FS
+
+	// Export to this path ON THE CLIENT MACHINE
+	dest: string
+}
+
+// Securely receive a secret from the client
+_#inputSecret: {
+	// Reference to the secret contents
+	// Use this by securely mounting it into a container.
+	// See universe.dagger.io/docker.#Run.mounts
+	// FIXME: `contents` field name causes confusion (not actually the secret contents..)
+	contents: #Secret
+
+	{
+		// Execute a command ON THE CLIENT MACHINE and read secret from standard output
+		command: [string, ...string] | string
+		// Execute command in an interactive terminal
+		//  for example to prompt for a passphrase
+		interactive: true | *false
+	} | {
+		// Read secret from a file ON THE CLIENT MACHINE
+		path: string
+	} | {
+		// Read secret from an environment variable ON THE CLIENT MACHINE
+		envvar: string
+	}
+}
+
+// Forward a network endpoint to and from the client
+_#proxyEndpoint: {
+	// Service endpoint can be proxied to action containers as unix sockets
+	// FIXME: should #Service be renamed to #ServiceEndpoint or #Endpoint? Naming things is hard...
+	endpoint: #Service
+
+	{
+		// Listen for connections ON THE CLIENT MACHINE, proxy to actions
+		listen: #Address
+	} | {
+		// Connect to a remote endpoint FROM THE CLIENT MACHINE, proxy to actions
+		connect: #Address
+	} | {
+		// Proxy to/from the contents of a file ON THE CLIENT MACHINE
+		filepath: string
+	} | {
+		// Proxy to/from standard input and output of a command ON THE CLIENT MACHINE
+		command: [string, ...string] | string
+	}
+}

stdlib/europa/dagger/types.cue

@@ -0,0 +1,36 @@
+package dagger
+
+import (
+	"alpha.dagger.io/europa/dagger/engine/spec/engine"
+)
+
+// A reference to a filesystem tree.
+// For example:
+//  - The root filesystem of a container
+//  - A source code repository
+//  - A directory containing binary artifacts
+// Rule of thumb: if it fits in a tar archive, it fits in a #FS.
+#FS: engine.#FS
+
+// A reference to an external secret, for example:
+//  - A password
+//  - A SSH private key
+//  - An API token
+// Secrets are never merged in the Cue tree. They can only be used
+// by a special filesystem mount designed to minimize leak risk.
+#Secret: engine.#Secret
+
+// A reference to a stream of bytes, for example:
+//  - The standard output or error stream of a command
+//  - The standard input stream of a command
+//  - The contents of a file or named pipe
+#Stream: engine.#Stream
+
+// A reference to a network service endpoint, for example:
+//  - A TCP or UDP port
+//  - A unix socket
+//  - An HTTPS endpoint
+#Service: engine.#Service
+
+// A network service address
+#Address: string & =~"^(tcp://|unix://|udp://).*"

stdlib/europa/dagger/utils.cue

@@ -0,0 +1,26 @@
+package dagger
+
+import (
+	"alpha.dagger.io/europa/dagger/engine/spec/engine"
+)
+
+// Select a subdirectory from a filesystem tree
+#Subdir: {
+	// Input tree
+	input: #FS
+
+	// Path of the subdirectory
+	// Example: "/build"
+	path: string
+
+	// Subdirectory tree
+	output: #FS & _copy.output
+
+	_copy: engine.#Copy & {
+		"input": engine.#Scratch.output
+		source: {
+			root:   input
+			"path": path
+		}
+	}
+}

stdlib/stdlib.go

@@ -14,12 +14,11 @@ import (
 
 var (
 	// FS contains the filesystem of the stdlib.
-	//go:embed **/*.cue **/*/*.cue
+	//go:embed **/*.cue **/*/*.cue europa/dagger/*.cue europa/dagger/engine/*.cue europa/dagger/engine/spec/*.cue europa/dagger/engine/spec/engine/*.cue
 	FS embed.FS
 
 	ModuleName    = "alpha.dagger.io"
-	PackageName   = fmt.Sprintf("%s/dagger", ModuleName)
-	EnginePackage = fmt.Sprintf("%s/engine", PackageName)
+	EnginePackage = fmt.Sprintf("%s/europa/dagger/engine", ModuleName)
 	Path          = path.Join("cue.mod", "pkg", ModuleName)
 	lockFilePath  = path.Join("cue.mod", "dagger.lock")
 )