Merge pull request #71 from blocklayerhq/export-fixes

Misc Export fixes

dagger/env.go

@@ -325,8 +325,6 @@ func (env *Env) Walk(ctx context.Context, fn EnvWalkFunc) (*Value, error) {
 			lg.Debug().Msg("cueflow task: filling result")
 			// Merge task value into output
 			var err error
-			// FIXME: does cueflow.Task.Value() contain only filled values,
-			// or base + filled?
 			out, err = out.MergePath(t.Value(), t.Path())
 			if err != nil {
 				lg.

dagger/gen.go

@@ -42,9 +42,8 @@ package dagger
 	#dagger: #ComponentConfig
 	...
 } | {
-	// Match embedded strings
+	// Match embedded scalars
-	// FIXME: match all embedded scalar types
+	bool | int | float | string | bytes
-	string
 	#dagger: #ComponentConfig
 }
 
@@ -68,7 +67,7 @@ package dagger
 	do: "export"
 	// Source path in the container
 	source: string
-	format: "json" | "yaml" | *"string" | "number" | "boolean"
+	format: "json" | "yaml" | *"string"
 }
 
 #Local: {

dagger/op.go

@@ -8,6 +8,7 @@ import (
 	"github.com/moby/buildkit/client/llb"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
+	"gopkg.in/yaml.v3"
 )
 
 type Op struct {
@@ -237,7 +238,8 @@ func (op *Op) Export(ctx context.Context, fs FS, out *Fillable) (FS, error) {
 		}
 	case "json":
 		var o interface{}
-		if err := json.Unmarshal(contents, &o); err != nil {
+		o, err := unmarshalAnything(contents, json.Unmarshal)
+		if err != nil {
 			return fs, err
 		}
 
@@ -247,6 +249,22 @@ func (op *Op) Export(ctx context.Context, fs FS, out *Fillable) (FS, error) {
 			Interface("contents", o).
 			Msg("exporting json")
 
+		if err := out.Fill(o); err != nil {
+			return fs, err
+		}
+	case "yaml":
+		var o interface{}
+		o, err := unmarshalAnything(contents, yaml.Unmarshal)
+		if err != nil {
+			return fs, err
+		}
+
+		log.
+			Ctx(ctx).
+			Debug().
+			Interface("contents", o).
+			Msg("exporting yaml")
+
 		if err := out.Fill(o); err != nil {
 			return fs, err
 		}
@@ -256,6 +274,25 @@ func (op *Op) Export(ctx context.Context, fs FS, out *Fillable) (FS, error) {
 	return fs, nil
 }
 
+type unmarshaller func([]byte, interface{}) error
+
+func unmarshalAnything(data []byte, fn unmarshaller) (interface{}, error) {
+	// unmarshalling a map into interface{} yields an error:
+	// "unsupported Go type for map key (interface {})"
+	// we want to attempt to unmarshal to a map[string]interface{} first
+	var oMap map[string]interface{}
+	if err := fn(data, &oMap); err == nil {
+		return oMap, nil
+	}
+
+	// If the previous attempt didn't work, we might be facing a scalar (e.g.
+	// bool).
+	// Try to unmarshal to interface{} directly.
+	var o interface{}
+	err := fn(data, &o)
+	return o, err
+}
+
 func (op *Op) Load(ctx context.Context, fs FS, out *Fillable) (FS, error) {
 	from, err := newExecutable(op.Get("from"))
 	if err != nil {

dagger/spec.cue

@@ -37,9 +37,8 @@ package dagger
 	#dagger: #ComponentConfig
 	...
 } | {
-	// Match embedded strings
+	// Match embedded scalars
-	// FIXME: match all embedded scalar types
+	bool | int | float | string | bytes
-	string
 	#dagger: #ComponentConfig
 }
 
@@ -63,7 +62,7 @@ package dagger
 	do: "export"
 	// Source path in the container
 	source: string
-	format: "json" | "yaml" | *"string" | "number" | "boolean"
+	format: "json" | "yaml" | *"string"
 }
 
 #Local: {

examples/tests/export/bool/main.cue

@@ -20,7 +20,7 @@ test: {
 			do: "export"
 			// Source path in the container
 			source: "/tmp/out"
-			format: "bool"
+			format: "json"
 		},
 	]
 }

examples/tests/export/float/main.cue

@@ -0,0 +1,25 @@
+package testing
+
+test: {
+	float
+
+	#dagger: compute: [
+		{
+			do:  "fetch-container"
+			ref: "alpine"
+		},
+		{
+			do: "exec"
+			args: ["sh", "-c", """
+				echo -123.5 > /tmp/out
+				""",
+			]
+		},
+		{
+			do: "export"
+			// Source path in the container
+			source: "/tmp/out"
+			format: "json"
+		},
+	]
+}

examples/tests/export/json/main.cue

@@ -1,6 +1,31 @@
 package testing
 
-test: #dagger: compute: [
+testScalar: {
+	bool
+
+	#dagger: compute: [
+		{
+			do:  "fetch-container"
+			ref: "alpine"
+		},
+		{
+			do: "exec"
+			args: ["sh", "-c", """
+				echo true > /tmp/out
+				""",
+			]
+			dir: "/"
+		},
+		{
+			do: "export"
+			// Source path in the container
+			source: "/tmp/out"
+			format: "json"
+		},
+	]
+}
+
+testMap: #dagger: compute: [
 	{
 		do:  "fetch-container"
 		ref: "alpine"
@@ -20,3 +45,29 @@ test: #dagger: compute: [
 		format: "json"
 	},
 ]
+
+// FIXME: lists are currently broken
+// testList: {
+//  [...string]
+
+//  #dagger: compute: [
+//   {
+//    do:  "fetch-container"
+//    ref: "alpine"
+//   },
+//   {
+//    do: "exec"
+//    args: ["sh", "-c", """
+//     echo '["milk", "pumpkin pie", "eggs", "juice"]' > /tmp/out
+//     """,
+//    ]
+//    dir: "/"
+//   },
+//   {
+//    do: "export"
+//    // Source path in the container
+//    source: "/tmp/out"
+//    format: "json"
+//   },
+//  ]
+// }

examples/tests/export/number/main.cue

@@ -19,7 +19,7 @@ test: {
 			do: "export"
 			// Source path in the container
 			source: "/tmp/out"
-			format: "number"
+			format: "json"
 		},
 	]
 }

examples/tests/export/yaml/main.cue

@@ -1,6 +1,9 @@
 package testing
 
-test: #dagger: compute: [
+testScalar: {
+	bool
+
+	#dagger: compute: [
 		{
 			do:  "fetch-container"
 			ref: "alpine"
@@ -8,8 +11,58 @@ test: #dagger: compute: [
 		{
 			do: "exec"
 			args: ["sh", "-c", """
-			echo "--- # Shopping list
+				echo true > /tmp/out
-			[milk, pumpkin pie, eggs, juice]" > /tmp/out
+				""",
+			]
+			// XXX Blocked by https://github.com/blocklayerhq/dagger/issues/19
+			dir: "/"
+		},
+		{
+			do: "export"
+			// Source path in the container
+			source: "/tmp/out"
+			format: "yaml"
+		},
+	]
+}
+
+// FIXME: lists are currently broken
+// testList: {
+//  [...string]
+
+//  #dagger: compute: [
+//   {
+//    do:  "fetch-container"
+//    ref: "alpine"
+//   },
+//   {
+//    do: "exec"
+//    args: ["sh", "-c", """
+//     echo "--- # Shopping list
+//     [milk, pumpkin pie, eggs, juice]" > /tmp/out
+//     """,
+//    ]
+//    // XXX Blocked by https://github.com/blocklayerhq/dagger/issues/19
+//    dir: "/"
+//   },
+//   {
+//    do: "export"
+//    // Source path in the container
+//    source: "/tmp/out"
+//    format: "yaml"
+//   },
+//  ]
+// }
+
+testMap: #dagger: compute: [
+	{
+		do:  "fetch-container"
+		ref: "alpine"
+	},
+	{
+		do: "exec"
+		args: ["sh", "-c", """
+			echo something: something > /tmp/out
 			""",
 		]
 	},

examples/tests/test.sh

@@ -129,7 +129,7 @@ test::exec(){
 }
 
 test::export(){
-  test::one "Export: json" --exit=0 --stdout='{"test":{"something":"something"}}' \
+  test::one "Export: json" --exit=0 --stdout='{"testMap":{"something":"something"},"testScalar":true}' \
       "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/export/json
 
   test::one "Export: string" --exit=0 --stdout='{"test":"something"}' \
@@ -150,11 +150,16 @@ test::export(){
   test::one "Export: invalid path" --exit=1 --stdout= \
       "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/export/invalid/path
 
-  disable test::one "Export: number (FIXME https://github.com/blocklayerhq/dagger/issues/36)" --exit=0 --stdout='{"test": -123.5}' \
+  test::one "Export: number" --exit=0 --stdout='{"test":-123.5}' \
+      "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/export/float
+
+  disable test::one "Export: number (FIXME: https://github.com/blocklayerhq/dagger/issues/96)" --exit=0 --stdout='{"test":-123.5}' \
       "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/export/number
-  disable test::one "Export: yaml (FIXME https://github.com/blocklayerhq/dagger/issues/36)" --exit=0 --stdout='XXXXXX' \
+
+  test::one "Export: yaml" --exit=0 --stdout='{"testMap":{"something":"something"},"testScalar":true}' \
       "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/export/yaml
-  disable test::one "Export: bool (FIXME https://github.com/blocklayerhq/dagger/issues/35)" --exit=0 --stdout='{"test": false}' \
+
+  test::one "Export: bool" --exit=0 --stdout='{"test":true}' \
       "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/export/bool
 }
 

go.mod

@@ -18,7 +18,7 @@ require (
 	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
 	golang.org/x/term v0.0.0-20201117132131-f5c789dd3221
 	golang.org/x/tools v0.1.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86
 )
 
 replace (