stdlib: os package

Signed-off-by: Solomon Hykes <solomon@dagger.io>

stdlib/dagger/dagger.cue

@@ -1,16 +1,14 @@
 package dagger
 
 import (
-	"dagger.io/dagger/op"
 )
+// "dagger.io/dagger/op"
 
 // An artifact such as source code checkout, container image, binary archive...
 // May be passed as user input, or computed by a buildkit pipeline
-#Artifact: {
+
-	#up: [...op.#Op]
+// FIXME (perf). See https://github.com/dagger/dagger/issues/445
-	_
+#Artifact: _
-	...
-}
 
 // Secret value
 // FIXME: currently aliased as a string to mark secrets

stdlib/dagger/op/op.cue

@@ -48,7 +48,10 @@ package op
 	// `true` means also ignoring the mount cache volumes
 	always?: true | *false
 	dir:     string | *"/"
-	mount: [string]: "tmpfs" | "cache" | {from: _, path: string | *"/"}
+	// FIXME (perf): complex schema in low-level ops causes explosive perf issues
+	//    see https://github.com/dagger/dagger/issues/445
+	// mount: [string]: "tmpfs" | "cache" | {from: _, path: string | *"/"}
+	mount: [string]: _
 }
 
 #DockerLogin: {

stdlib/docker/docker.cue

@@ -17,22 +17,21 @@ import (
 
 }
 
-// Build a Docker image from source, using included Dockerfile
+// Pull a docker container
-// FIXME: DEPRECATED by #Build
+#Pull: {
-#ImageFromSource: #Build
+	// Remote ref (example: "index.docker.io/alpine:latest")
-
+	from: string
-// Fetch an image from a remote registry
-#ImageFromRegistry: {
-	ref: string
 
 	#up: [
-		op.#FetchContainer & {
+		op.#FetchContainer & {ref: from},
-			"ref": ref
-		},
 	]
 }
 
+// FIXME: #Push
+// FIXME: #Run
+
 // Build a Docker image from the provided Dockerfile contents
+// FIXME: incorporate into #Build
 #ImageFromDockerfile: {
 	dockerfile: string
 	context:    dagger.#Artifact

stdlib/go/go.cue

@@ -4,7 +4,7 @@ import (
 	"dagger.io/dagger"
 	"dagger.io/dagger/op"
 
-	"dagger.io/docker"
+	"dagger.io/os"
 )
 
 // A standalone go environment
@@ -13,15 +13,14 @@ import (
 	version: *"1.16" | string
 	source:  dagger.#Artifact
 
-	docker.#Container & {
+	os.#Container & {
 		env: {
 			GOMODCACHE:  volume.goCache.dest
 			CGO_ENABLED: "0"
 		}
 
-		image: docker.#ImageFromRegistry & {
+		from: "docker.io/golang:\(version)-alpine"
-			ref: "docker.io/golang:\(version)-alpine"
+
-		}
 		volume: {
 			goSource: {
 				from: source

stdlib/io/io.cue

@@ -21,7 +21,6 @@ package io
 		format: "string" | "json" | "yaml" | "lines"
 		data: {
 			string
-			...
 		}
 	}
 	...

stdlib/js/react/react.cue

@@ -4,11 +4,12 @@ import (
 	"strings"
 
 	"dagger.io/dagger"
-	"dagger.io/dagger/op"
 	"dagger.io/alpine"
+	"dagger.io/os"
 )
 
 // A ReactJS application
+// FIXME: move this to a 'yarn' package for clarity
 #App: {
 	// Application source code
 	source: dagger.#Artifact
@@ -30,32 +31,26 @@ import (
 		script: string | *"build"
 	}
 
-	build: #up: [
+	build: os.#Dir & {
-		op.#Load & {
+		from: ctr
-			from: alpine.#Image & {
+		path: "/build"
-				package: bash: "=~5.1"
-				package: yarn: "=~1.22"
 	}
-		},
+
-		op.#Exec & {
+	ctr: os.#Container & {
-			args: [
+		image: alpine.#Image & {
-				"/bin/bash",
+			package: {
-				"--noprofile",
+				bash: "=~5.1"
-				"--norc",
+				yarn: "=~1.22"
-				"-eo",
+			}
-				"pipefail",
+		}
-				"-c",
+		shell: path: "/bin/bash"
-				"""
+		command: """
 			[ -n "$ENVFILE_NAME" ] && echo "$ENVFILE" > "$ENVFILE_NAME"
 			yarn install --production false
 			yarn run "$YARN_BUILD_SCRIPT"
 			mv "$YARN_BUILD_DIRECTORY" /build
-					""",
+			"""
-			]
+		"env": env & {
-			if env != _|_ {
-				"env": env
-			}
-			"env": {
 			YARN_BUILD_SCRIPT:    yarn.script
 			YARN_CACHE_FOLDER:    "/cache/yarn"
 			YARN_BUILD_DIRECTORY: yarn.buildDir
@@ -65,13 +60,7 @@ import (
 			}
 		}
 		dir: "/src"
-			mount: {
+		mount: "/src": from: source
-				"/src": from: source
+		cache: "/cache/yarn": true
-				"/cache/yarn": "cache"
 	}
-		},
-		op.#Subdir & {
-			dir: "/build"
-		},
-	]
 }

stdlib/netlify/netlify.cue

@@ -3,7 +3,7 @@ package netlify
 import (
 	"dagger.io/dagger"
 	"dagger.io/alpine"
-	"dagger.io/dagger/op"
+	"dagger.io/os"
 )
 
 // A Netlify account
@@ -34,36 +34,44 @@ import (
 	create: bool | *true
 
 	// Website url
-	url: string
+	url: {
+		os.#File & {
+			from: ctr
+			path: "/netlify/url"
+		}
+	}.read.data
 
 	// Unique Deploy URL
-	deployUrl: string
+	deployUrl: {
+		os.#File & {
+			from: ctr
+			path: "/netlify/deployUrl"
+		}
+	}.read.data
 
 	// Logs URL for this deployment
-	logsUrl: string
+	logsUrl: {
-
+		os.#File & {
-	#up: [
+			from: ctr
-		op.#Load & {
+			path: "/netlify/logsUrl"
-			from: alpine.#Image & {
-				package: bash: "=~5.1"
-				package: jq:   "=~1.6"
-				package: curl: "=~7.76"
-				package: yarn: "=~1.22"
 		}
-		},
+	}.read.data
-		op.#Exec & {
+
-			args: ["yarn", "global", "add", "netlify-cli@2.47.0"]
+	ctr: os.#Container & {
-		},
+		image: alpine.#Image & {
-		op.#Exec & {
+			package: {
-			args: [
+				bash: "=~5.1"
-				"/bin/bash",
+				jq:   "=~1.6"
-				"--noprofile",
+				curl: "=~7.76"
-				"--norc",
+				yarn: "=~1.22"
-				"-eo",
+			}
-				"pipefail",
+		}
-				"-c",
+		setup: [
-				#code,
+			"yarn global add netlify-cli@2.47.0",
 		]
+		// set in netlify.sh.cue
+		// FIXME: use embedding once cue supports it
+		command: _
 		env: {
 			NETLIFY_SITE_NAME: name
 			if (create) {
@@ -77,10 +85,5 @@ import (
 		}
 		dir: "/src"
 		mount: "/src": from: contents
-		},
+	}
-		op.#Export & {
-			source: "/output.json"
-			format: "json"
-		},
-	]
 }

stdlib/netlify/netlify.sh.cue

@@ -1,6 +1,6 @@
 package netlify
 
-#code: #"""
+#Site: ctr: command: #"""
 	create_site() {
 	    url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites"
 
@@ -41,9 +41,10 @@ package netlify
 	deployUrl=$(</tmp/stdout sed -n -e 's/^Unique Deploy URL:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n')
 	logsUrl=$(</tmp/stdout sed -n -e 's/^Logs:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n')
 
-	jq -n \
+	# Write output files
-		--arg url "$url" \
+	mkdir -p /netlify
-		--arg deployUrl "$deployUrl" \
+	echo "$url" > /netlify/url
-		--arg logsUrl "$logsUrl" \
+	echo "$deployUrl" > /netlify/deployUrl
-		'{url: $url, deployUrl: $deployUrl, logsUrl: $logsUrl}' > /output.json
+	echo "$logsUrl" > /netlify/logsUrl
+
 	"""#

stdlib/os/container.cue

@@ -1,7 +1,4 @@
-// docker: build and run Docker containers
+package os
-// https://docker.com
-
-package docker
 
 import (
 	"strings"
@@ -18,11 +15,17 @@ import (
 //   see https://github.com/dagger/dagger/issues/304
 #DefaultImage: alpine.#Image.#up
 
-// Run a Docker container
+// Built-in container implementation, using buildkit
 #Container: {
 
 	// Container image
 	image: dagger.#Artifact | *#DefaultImage
+	//     {
+	//      // Optionally fetch a remote image
+	//      // eg. "index.docker.io/alpine"
+	//      from: string
+	//      image: #up: [op.#FetchContainer & { "ref": from }]
+	//     } | {}
 
 	// Independently cacheable setup commands
 	setup: [...string]
@@ -36,43 +39,28 @@ import (
 	// Directory in which the command is executed
 	dir: string | *"/"
 
-	// Directory to expose as the output.
-	// By default the root filesystem is the output.
-	outputDir: string | *"/"
-
 	// If true, the command is never cached.
 	// (false by default).
 	always: true | *false
 
-	// External volumes. There are 4 types:
+	// Copy contents from other artifacts
-	//
+	copy: [string]: from: dagger.#Artifact
-	// 1. "mount": mount any artifact.
-	//     Changes are not included in the final output.
-	//
-	// 2. "copy": copy any artifact.
-	//     Changes are included in the final output.
-	//
-	// 3. "tmpfs": create a temporary directory.
-	//
-	// 4. "cache": create a persistent cache diretory.
-	//
-	volume: [name=string]: {
-		// Destination path
-		dest: string | *"/"
 
-		*{
+	// Mount contents from other artifacts.
-			type:   "mount"
+	// Mount is active when executing `command`, but not `setup`.
-			from:   dagger.#Artifact
+
-			source: string | *"/"
+	mount: [string]: {
-		} | {
+		// FIXME(perf) should be #Artifact. See https://github.com/dagger/dagger/issues/445
-			type:   "copy"
+		from: {...}
-			from:   dagger.#Artifact
+		// FIXME: support source path
-			source: string | *"/"
-		} | {
-			type: "tmpfs" | "cache"
-		}
 	}
 
+	// Mount persistent cache directories
+	cache: [string]: true
+
+	// Mount temporary directories
+	tmpfs: [string]: true
+
 	// Configure the shell which executes all commands.
 	shell: {
 		// Path of the shell to execute
@@ -94,20 +82,14 @@ import (
 	}
 	env: PATH: string | *strings.Join([ for p, v in shell.search if v {p}], ":")
 
-	// Export values from the container to the cue configuration
-	export: *null | {
-		source: string
-		format: op.#Export.format
-	}
-
 	#up: [
 		op.#Load & {from: image},
 		// Copy volumes with type=copy
-		for _, v in volume if v.type == "copy" {
+		for dest, o in copy {
 			op.#Copy & {
-				from: v.from
+				"dest": dest
-				dest: v.dest
+				from:   o.from
-				src:  v.source
+				// FIXME: support source path
 			}
 		},
 		// Execute setup commands, without volumes
@@ -126,33 +108,19 @@ import (
 				"env":    env
 				"dir":    dir
 				"always": always
-				mount: {
+				"mount": {
-					// FIXME: fix perf issue
+					for dest, o in mount {
-					for _, v in volume if v.type == "cache" {
+						"\(dest)": from: o.from
-						"\(v.dest)": "cache"
+						// FIXME: support source path
 					}
-					for _, v in volume if v.type == "tmpfs" {
+					for dest in cache {
-						"\(v.dest)": "tmpfs"
+						"\(dest)": "cache"
 					}
-					for _, v in volume if v.type == "mount" {
+					for dest in tmpfs {
-						"\(v.dest)": {
+						"\(dest)": "tmpfs"
-							from: v.from
-							path: v.source
 					}
 				}
 			}
-			}
-		},
-		// FIXME: is subdir deprecated by dagger.io/io.#Dir ?
-		op.#Subdir & {
-			dir: outputDir
-		},
-		// FIXME: is export deprecated by dagger.io/io.#File ?
-		if export != null {
-			op.#Export & {
-				source: export.source
-				format: export.format
-			}
 		},
 	]
 }

stdlib/os/dir.cue

@@ -1,13 +1,23 @@
 package os
 
 import (
-	"dagger.io/io"
-
 	"dagger.io/dagger"
 	"dagger.io/dagger/op"
 )
 
-#Dir: io.#Dir & {
+#Dir: {
+	from: dagger.#Artifact
+	path: string | *"/"
+
+	#up: [
+		op.#Load & {"from": from},
+		op.#Subdir & {
+			dir: path
+		},
+	]
+}
+
+#ReadDir: {
 	from: dagger.#Artifact
 	path: string
 
@@ -32,34 +42,3 @@ import (
 		]
 	}
 }
-
-#File: {
-	from: dagger.#Artifact
-	path: string
-
-	io.#Reader & {
-		read: {
-			format: _
-			data: {
-				_
-				#up: [
-					op.#Load & {"from":   from},
-					op.#Export & {source: path, "format": format},
-				]
-			}
-		}
-	}
-
-	io.#Writer & {
-		write: *null | {
-			data: _
-			#up: [
-				op.#Load & {"from": from},
-				op.#WriteFile & {
-					dest:     path
-					contents: data
-				},
-			]
-		}
-	}
-}

stdlib/os/file.cue

@@ -0,0 +1,36 @@
+package os
+
+import (
+	"dagger.io/dagger"
+	"dagger.io/dagger/op"
+)
+
+// Built-in file implementation, using buildkit
+#File: {
+	from: dagger.#Artifact
+	path: string
+
+	read: {
+		// FIXME: support different data schemas for different formats
+		format: "string"
+		data: {
+			string
+			#up: [
+				op.#Load & {"from":   from},
+				op.#Export & {source: path, "format": format},
+			]
+		}
+	}
+
+	write: *null | {
+		// FIXME: support encoding in different formats
+		data: string
+		#up: [
+			op.#Load & {"from": from},
+			op.#WriteFile & {
+				dest:     path
+				contents: data
+			},
+		]
+	}
+}

tests/stdlib/js/react/react.cue

@@ -4,7 +4,7 @@ import (
 	"dagger.io/dagger"
 	"dagger.io/js/react"
 	"dagger.io/alpine"
-	"dagger.io/docker"
+	"dagger.io/os"
 )
 
 TestData: dagger.#Artifact
@@ -14,26 +14,13 @@ TestReact: {
 		source: TestData
 	}
 
-	test: docker.#Container & {
+	test: os.#Container & {
 		image: alpine.#Image & {
 			package: bash: "=5.1.0-r0"
 		}
-		volume: build: {
+		mount: "/build": from: app.build
-			from: app.build
-			dest: "/build"
-		}
 		command: """
 			test "$(cat /build/test)" = "output"
 			"""
-		shell: {
-			path: "/bin/bash"
-			args: [
-				"--noprofile",
-				"--norc",
-				"-eo",
-				"pipefail",
-				"-c",
-			]
-		}
 	}
 }