From c6e010d4f075215c34f97411ccd855ece955c7ca Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Mon, 25 Jan 2021 15:07:54 -0800 Subject: [PATCH] Misc Export fixes - Temporarily disable export of base.cue as it causes merge errors at the end of compute - Fixes for JSON export for Scalar and Lists - Add YAML export - Removed BOOL and NUMBER support, using JSON for now - Re-enabled all export tests Fixes #36 Signed-off-by: Andrea Luzzardi --- dagger/env.go | 2 - dagger/gen.go | 7 ++-- dagger/op.go | 39 +++++++++++++++++- dagger/spec.cue | 7 ++-- examples/tests/export/bool/main.cue | 2 +- examples/tests/export/float/main.cue | 25 ++++++++++++ examples/tests/export/json/main.cue | 53 +++++++++++++++++++++++- examples/tests/export/number/main.cue | 2 +- examples/tests/export/yaml/main.cue | 59 +++++++++++++++++++++++++-- examples/tests/test.sh | 13 ++++-- go.mod | 2 +- 11 files changed, 189 insertions(+), 22 deletions(-) create mode 100644 examples/tests/export/float/main.cue diff --git a/dagger/env.go b/dagger/env.go index 80cd92da..a9b87dd1 100644 --- a/dagger/env.go +++ b/dagger/env.go @@ -325,8 +325,6 @@ func (env *Env) Walk(ctx context.Context, fn EnvWalkFunc) (*Value, error) { lg.Debug().Msg("cueflow task: filling result") // Merge task value into output var err error - // FIXME: does cueflow.Task.Value() contain only filled values, - // or base + filled? out, err = out.MergePath(t.Value(), t.Path()) if err != nil { lg. diff --git a/dagger/gen.go b/dagger/gen.go index c95bb755..a0e44ad1 100644 --- a/dagger/gen.go +++ b/dagger/gen.go @@ -42,9 +42,8 @@ package dagger #dagger: #ComponentConfig ... } | { - // Match embedded strings - // FIXME: match all embedded scalar types - string + // Match embedded scalars + bool | int | float | string | bytes #dagger: #ComponentConfig } @@ -68,7 +67,7 @@ package dagger do: "export" // Source path in the container source: string - format: "json" | "yaml" | *"string" | "number" | "boolean" + format: "json" | "yaml" | *"string" } #Local: { diff --git a/dagger/op.go b/dagger/op.go index 3cc2335b..aa5a4243 100644 --- a/dagger/op.go +++ b/dagger/op.go @@ -8,6 +8,7 @@ import ( "github.com/moby/buildkit/client/llb" "github.com/pkg/errors" "github.com/rs/zerolog/log" + "gopkg.in/yaml.v3" ) type Op struct { @@ -237,7 +238,8 @@ func (op *Op) Export(ctx context.Context, fs FS, out *Fillable) (FS, error) { } case "json": var o interface{} - if err := json.Unmarshal(contents, &o); err != nil { + o, err := unmarshalAnything(contents, json.Unmarshal) + if err != nil { return fs, err } @@ -247,6 +249,22 @@ func (op *Op) Export(ctx context.Context, fs FS, out *Fillable) (FS, error) { Interface("contents", o). Msg("exporting json") + if err := out.Fill(o); err != nil { + return fs, err + } + case "yaml": + var o interface{} + o, err := unmarshalAnything(contents, yaml.Unmarshal) + if err != nil { + return fs, err + } + + log. + Ctx(ctx). + Debug(). + Interface("contents", o). + Msg("exporting yaml") + if err := out.Fill(o); err != nil { return fs, err } @@ -256,6 +274,25 @@ func (op *Op) Export(ctx context.Context, fs FS, out *Fillable) (FS, error) { return fs, nil } +type unmarshaller func([]byte, interface{}) error + +func unmarshalAnything(data []byte, fn unmarshaller) (interface{}, error) { + // unmarshalling a map into interface{} yields an error: + // "unsupported Go type for map key (interface {})" + // we want to attempt to unmarshal to a map[string]interface{} first + var oMap map[string]interface{} + if err := fn(data, &oMap); err == nil { + return oMap, nil + } + + // If the previous attempt didn't work, we might be facing a scalar (e.g. + // bool). + // Try to unmarshal to interface{} directly. + var o interface{} + err := fn(data, &o) + return o, err +} + func (op *Op) Load(ctx context.Context, fs FS, out *Fillable) (FS, error) { from, err := newExecutable(op.Get("from")) if err != nil { diff --git a/dagger/spec.cue b/dagger/spec.cue index fa53d105..f7999f2f 100644 --- a/dagger/spec.cue +++ b/dagger/spec.cue @@ -37,9 +37,8 @@ package dagger #dagger: #ComponentConfig ... } | { - // Match embedded strings - // FIXME: match all embedded scalar types - string + // Match embedded scalars + bool | int | float | string | bytes #dagger: #ComponentConfig } @@ -63,7 +62,7 @@ package dagger do: "export" // Source path in the container source: string - format: "json" | "yaml" | *"string" | "number" | "boolean" + format: "json" | "yaml" | *"string" } #Local: { diff --git a/examples/tests/export/bool/main.cue b/examples/tests/export/bool/main.cue index 27de7231..f6ec951b 100644 --- a/examples/tests/export/bool/main.cue +++ b/examples/tests/export/bool/main.cue @@ -20,7 +20,7 @@ test: { do: "export" // Source path in the container source: "/tmp/out" - format: "bool" + format: "json" }, ] } diff --git a/examples/tests/export/float/main.cue b/examples/tests/export/float/main.cue new file mode 100644 index 00000000..b36be873 --- /dev/null +++ b/examples/tests/export/float/main.cue @@ -0,0 +1,25 @@ +package testing + +test: { + float + + #dagger: compute: [ + { + do: "fetch-container" + ref: "alpine" + }, + { + do: "exec" + args: ["sh", "-c", """ + echo -123.5 > /tmp/out + """, + ] + }, + { + do: "export" + // Source path in the container + source: "/tmp/out" + format: "json" + }, + ] +} diff --git a/examples/tests/export/json/main.cue b/examples/tests/export/json/main.cue index 713b9c24..c557a665 100644 --- a/examples/tests/export/json/main.cue +++ b/examples/tests/export/json/main.cue @@ -1,6 +1,31 @@ package testing -test: #dagger: compute: [ +testScalar: { + bool + + #dagger: compute: [ + { + do: "fetch-container" + ref: "alpine" + }, + { + do: "exec" + args: ["sh", "-c", """ + echo true > /tmp/out + """, + ] + dir: "/" + }, + { + do: "export" + // Source path in the container + source: "/tmp/out" + format: "json" + }, + ] +} + +testMap: #dagger: compute: [ { do: "fetch-container" ref: "alpine" @@ -20,3 +45,29 @@ test: #dagger: compute: [ format: "json" }, ] + +// FIXME: lists are currently broken +// testList: { +// [...string] + +// #dagger: compute: [ +// { +// do: "fetch-container" +// ref: "alpine" +// }, +// { +// do: "exec" +// args: ["sh", "-c", """ +// echo '["milk", "pumpkin pie", "eggs", "juice"]' > /tmp/out +// """, +// ] +// dir: "/" +// }, +// { +// do: "export" +// // Source path in the container +// source: "/tmp/out" +// format: "json" +// }, +// ] +// } diff --git a/examples/tests/export/number/main.cue b/examples/tests/export/number/main.cue index 43dc865f..4ba8dbd8 100644 --- a/examples/tests/export/number/main.cue +++ b/examples/tests/export/number/main.cue @@ -19,7 +19,7 @@ test: { do: "export" // Source path in the container source: "/tmp/out" - format: "number" + format: "json" }, ] } diff --git a/examples/tests/export/yaml/main.cue b/examples/tests/export/yaml/main.cue index d9695047..07bdefb0 100644 --- a/examples/tests/export/yaml/main.cue +++ b/examples/tests/export/yaml/main.cue @@ -1,6 +1,60 @@ package testing -test: #dagger: compute: [ +testScalar: { + bool + + #dagger: compute: [ + { + do: "fetch-container" + ref: "alpine" + }, + { + do: "exec" + args: ["sh", "-c", """ + echo true > /tmp/out + """, + ] + // XXX Blocked by https://github.com/blocklayerhq/dagger/issues/19 + dir: "/" + }, + { + do: "export" + // Source path in the container + source: "/tmp/out" + format: "yaml" + }, + ] +} + +// FIXME: lists are currently broken +// testList: { +// [...string] + +// #dagger: compute: [ +// { +// do: "fetch-container" +// ref: "alpine" +// }, +// { +// do: "exec" +// args: ["sh", "-c", """ +// echo "--- # Shopping list +// [milk, pumpkin pie, eggs, juice]" > /tmp/out +// """, +// ] +// // XXX Blocked by https://github.com/blocklayerhq/dagger/issues/19 +// dir: "/" +// }, +// { +// do: "export" +// // Source path in the container +// source: "/tmp/out" +// format: "yaml" +// }, +// ] +// } + +testMap: #dagger: compute: [ { do: "fetch-container" ref: "alpine" @@ -8,8 +62,7 @@ test: #dagger: compute: [ { do: "exec" args: ["sh", "-c", """ - echo "--- # Shopping list - [milk, pumpkin pie, eggs, juice]" > /tmp/out + echo something: something > /tmp/out """, ] }, diff --git a/examples/tests/test.sh b/examples/tests/test.sh index 567aff84..78e11172 100755 --- a/examples/tests/test.sh +++ b/examples/tests/test.sh @@ -129,7 +129,7 @@ test::exec(){ } test::export(){ - test::one "Export: json" --exit=0 --stdout='{"test":{"something":"something"}}' \ + test::one "Export: json" --exit=0 --stdout='{"testMap":{"something":"something"},"testScalar":true}' \ "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/export/json test::one "Export: string" --exit=0 --stdout='{"test":"something"}' \ @@ -150,11 +150,16 @@ test::export(){ test::one "Export: invalid path" --exit=1 --stdout= \ "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/export/invalid/path - disable test::one "Export: number (FIXME https://github.com/blocklayerhq/dagger/issues/36)" --exit=0 --stdout='{"test": -123.5}' \ + test::one "Export: number" --exit=0 --stdout='{"test":-123.5}' \ + "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/export/float + + disable test::one "Export: number (FIXME: https://github.com/blocklayerhq/dagger/issues/96)" --exit=0 --stdout='{"test":-123.5}' \ "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/export/number - disable test::one "Export: yaml (FIXME https://github.com/blocklayerhq/dagger/issues/36)" --exit=0 --stdout='XXXXXX' \ + + test::one "Export: yaml" --exit=0 --stdout='{"testMap":{"something":"something"},"testScalar":true}' \ "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/export/yaml - disable test::one "Export: bool (FIXME https://github.com/blocklayerhq/dagger/issues/35)" --exit=0 --stdout='{"test": false}' \ + + test::one "Export: bool" --exit=0 --stdout='{"test":true}' \ "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/export/bool } diff --git a/go.mod b/go.mod index ab846694..23b7e162 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 golang.org/x/tools v0.1.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 // indirect + gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 ) replace (