From c7ffdf788fb5f97f47db45b8a94e1e96e396d36c Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 13 Jan 2022 17:43:34 -0800 Subject: [PATCH 1/7] WIP: Fix Europa yarn Signed-off-by: Andrea Luzzardi --- pkg/dagger.io/cue.mod/module.cue | 1 + pkg/dagger.io/dagger/engine/types.cue | 6 +- pkg/universe.dagger.io/cue.mod/pkg/.gitignore | 5 ++ .../cue.mod/pkg/alpha.dagger.io | 1 - pkg/universe.dagger.io/docker/build.cue | 1 + .../{tests => test}/testdata/package.json | 0 .../{tests => test}/testdata2/package.json | 0 .../yarn/test/yarn-test.cue | 82 +++++++++++++++++++ .../yarn/tests/simple/simple.cue | 11 --- pkg/universe.dagger.io/yarn/yarn.cue | 60 ++++++++++---- plancontext/fs.go | 2 +- 11 files changed, 137 insertions(+), 32 deletions(-) create mode 100644 pkg/dagger.io/cue.mod/module.cue create mode 100644 pkg/universe.dagger.io/cue.mod/pkg/.gitignore delete mode 120000 pkg/universe.dagger.io/cue.mod/pkg/alpha.dagger.io rename pkg/universe.dagger.io/yarn/{tests => test}/testdata/package.json (100%) rename pkg/universe.dagger.io/yarn/{tests => test}/testdata2/package.json (100%) create mode 100644 pkg/universe.dagger.io/yarn/test/yarn-test.cue delete mode 100644 pkg/universe.dagger.io/yarn/tests/simple/simple.cue diff --git a/pkg/dagger.io/cue.mod/module.cue b/pkg/dagger.io/cue.mod/module.cue new file mode 100644 index 00000000..b2757079 --- /dev/null +++ b/pkg/dagger.io/cue.mod/module.cue @@ -0,0 +1 @@ +module: "dagger.io" diff --git a/pkg/dagger.io/dagger/engine/types.cue b/pkg/dagger.io/dagger/engine/types.cue index 4c13245c..15670df0 100644 --- a/pkg/dagger.io/dagger/engine/types.cue +++ b/pkg/dagger.io/dagger/engine/types.cue @@ -7,7 +7,7 @@ package engine // - A directory containing binary artifacts // Rule of thumb: if it fits in a tar archive, it fits in a #FS. #FS: { - $dagger: fs: _id: string | null + $dagger: fs: _id: !="" | null } // A reference to an external secret, for example: @@ -17,7 +17,7 @@ package engine // Secrets are never merged in the Cue tree. They can only be used // by a special filesystem mount designed to minimize leak risk. #Secret: { - $dagger: secret: _id: string + $dagger: secret: _id: !="" } // A reference to a network service endpoint, for example: @@ -25,5 +25,5 @@ package engine // - A unix or npipe socket // - An HTTPS endpoint #Service: { - $dagger: service: _id: string + $dagger: service: _id: !="" } diff --git a/pkg/universe.dagger.io/cue.mod/pkg/.gitignore b/pkg/universe.dagger.io/cue.mod/pkg/.gitignore new file mode 100644 index 00000000..331f139d --- /dev/null +++ b/pkg/universe.dagger.io/cue.mod/pkg/.gitignore @@ -0,0 +1,5 @@ +# generated by dagger +dagger.lock +alpha.dagger.io +dagger.io +universe.dagger.io \ No newline at end of file diff --git a/pkg/universe.dagger.io/cue.mod/pkg/alpha.dagger.io b/pkg/universe.dagger.io/cue.mod/pkg/alpha.dagger.io deleted file mode 120000 index 961afc64..00000000 --- a/pkg/universe.dagger.io/cue.mod/pkg/alpha.dagger.io +++ /dev/null @@ -1 +0,0 @@ -../../../stdlib \ No newline at end of file diff --git a/pkg/universe.dagger.io/docker/build.cue b/pkg/universe.dagger.io/docker/build.cue index 16e0724b..7ff5c8a8 100644 --- a/pkg/universe.dagger.io/docker/build.cue +++ b/pkg/universe.dagger.io/docker/build.cue @@ -10,6 +10,7 @@ import ( steps: [#Step, ...#Step] output: #Image + // Generate build DAG from linerar steps _dag: { for idx, step in steps { diff --git a/pkg/universe.dagger.io/yarn/tests/testdata/package.json b/pkg/universe.dagger.io/yarn/test/testdata/package.json similarity index 100% rename from pkg/universe.dagger.io/yarn/tests/testdata/package.json rename to pkg/universe.dagger.io/yarn/test/testdata/package.json diff --git a/pkg/universe.dagger.io/yarn/tests/testdata2/package.json b/pkg/universe.dagger.io/yarn/test/testdata2/package.json similarity index 100% rename from pkg/universe.dagger.io/yarn/tests/testdata2/package.json rename to pkg/universe.dagger.io/yarn/test/testdata2/package.json diff --git a/pkg/universe.dagger.io/yarn/test/yarn-test.cue b/pkg/universe.dagger.io/yarn/test/yarn-test.cue new file mode 100644 index 00000000..014f238a --- /dev/null +++ b/pkg/universe.dagger.io/yarn/test/yarn-test.cue @@ -0,0 +1,82 @@ +package yarn + +import ( + "dagger.io/dagger" + "dagger.io/dagger/engine" + + "universe.dagger.io/yarn" + // "universe.dagger.io/alpine" + // "universe.dagger.io/bash" +) + +dagger.#Plan & { + inputs: { + directories: { + testdata: path: "./testdata" + testdata2: path: "./testdata2" + } + } + + actions: { + TestReact: { + cache: engine.#CacheDir & { + id: "yarn cache" + } + + pkg: yarn.#Build & { + source: inputs.directories.testdata.contents + "cache": cache + } + + _image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + // FIXME: use bash.#Script + test: engine.#Exec & { + input: _image.output + mounts: build: { + dest: "/build" + contents: pkg.output + } + args: [ + "sh", "-c", + #""" + test "$(cat /build/test)" = "output" + """# + ] + } + } + + // FIXME: re-enable? + // TestSecretsAndFile: { + // pkg: #Package & { + // source: inputs.directories.testdata2 + // writeEnvFile: "/.env" + // env: { + // one: "one" + // two: "two" + // } + // secrets: { + // secretone: dagger.#Secret @dagger(input) + // secretwo: dagger.#Secret @dagger(input) + // } + // } + + // test: os.#Container & { + // image: alpine.#Image & { + // package: bash: true + // } + // shell: path: "/bin/bash" + // mount: "/build": from: pkg.build + // command: """ + // content="$(cat /build/env)" + // [[ "${content}" = *"SECRETONE="* ]] && \\ + // [[ "${content}" = *"SECRETWO="* ]] && \\ + // [[ "${content}" = *"ONE=one"* ]] && \\ + // [[ "${content}" = *"TWO=two"* ]] + // """ + // } + // } + } +} \ No newline at end of file diff --git a/pkg/universe.dagger.io/yarn/tests/simple/simple.cue b/pkg/universe.dagger.io/yarn/tests/simple/simple.cue deleted file mode 100644 index 19d771b1..00000000 --- a/pkg/universe.dagger.io/yarn/tests/simple/simple.cue +++ /dev/null @@ -1,11 +0,0 @@ -package yarn - -import ( - "dagger.io/dagger/engine" -) - -b: #Build & { - source: engine.#Scratch -} - -out: b.output diff --git a/pkg/universe.dagger.io/yarn/yarn.cue b/pkg/universe.dagger.io/yarn/yarn.cue index 6f0d839c..4cac4723 100644 --- a/pkg/universe.dagger.io/yarn/yarn.cue +++ b/pkg/universe.dagger.io/yarn/yarn.cue @@ -7,7 +7,8 @@ import ( "dagger.io/dagger" "dagger.io/dagger/engine" - "universe.dagger.io/alpine" + // "universe.dagger.io/alpine" + "universe.dagger.io/docker" "universe.dagger.io/bash" ) @@ -30,28 +31,54 @@ import ( // Run this yarn script script: string | *"build" + // Fix for shadowing issues + let yarnScript = script + + // FIXME: `CacheDir` must be passed by the caller? + cache: engine.#CacheDir + // Optional arguments for the script args: [...string] | *[] // Secret variables + // FIXME: not implemented. Are they needed? secrets: [string]: dagger.#Secret // Yarn version yarnVersion: *"=~1.22" | string + // FIXME: trouble getting docker.#Build to work (cueflow task dependencies not working) + alpine: docker.#Pull & { + source: "index.docker.io/alpine:3.13.5@sha256:69e70a79f2d41ab5d637de98c1e0b055206ba40a8145e7bddb55ccc04e13cf8f" + } + yarnImage: docker.#Run & { + image: alpine.image + script: """ + apk add -U --no-cache bash yarn + """ + } + // Run yarn in a containerized build environment command: bash.#Run & { - *{ - image: (alpine.#Build & { - bash: version: "=~5.1" - yarn: version: yarnVersion - }).image - env: CUSTOM_IMAGE: "0" - } | { - env: CUSTOM_IMAGE: "1" - } + // FIXME: not working? + // *{ + // _image: alpine.#Build & { + // packages: { + // bash: version: "=~5.1" + // yarn: version: yarnVersion + // } + // } - script: """ + // image: _image.output + // env: CUSTOM_IMAGE: "0" + // } | { + // env: CUSTOM_IMAGE: "1" + // } + + image: docker.#Image & yarnImage.output + + + script: #""" # Create $ENVFILE_NAME file if set [ -n "$ENVFILE_NAME" ] && echo "$ENVFILE" > "$ENVFILE_NAME" @@ -60,12 +87,12 @@ import ( opts=( $(echo $YARN_ARGS) ) yarn --cwd "$YARN_CWD" run "$YARN_BUILD_SCRIPT" ${opts[@]} mv "$YARN_BUILD_DIRECTORY" /build - """ + """# mounts: { "yarn cache": { dest: "/cache/yarn" - contents: engine.#CacheDir + contents: cache } "package source": { dest: "/src" @@ -74,10 +101,11 @@ import ( // FIXME: mount secrets } - output: directories: "/build": _ + // FIXME + directories: "/build": _ env: { - YARN_BUILD_SCRIPT: script + YARN_BUILD_SCRIPT: yarnScript YARN_ARGS: strings.Join(args, "\n") YARN_CACHE_FOLDER: "/cache/yarn" YARN_CWD: cwd @@ -92,5 +120,5 @@ import ( } // The final contents of the package after build - output: command.output.directories."/build".contents + output: command.directories."/build".contents } diff --git a/plancontext/fs.go b/plancontext/fs.go index ab6f09c6..f1213ce3 100644 --- a/plancontext/fs.go +++ b/plancontext/fs.go @@ -84,7 +84,7 @@ func (c *fsContext) FromValue(v *compiler.Value) (*FS, error) { id, err := v.LookupPath(fsIDPath).String() if err != nil { - return nil, fmt.Errorf("invalid FS %q: %w", v.Path(), err) + return nil, fmt.Errorf("invalid FS at path %q: %w", v.Path(), err) } fs, ok := c.store[id] From ce378d50950d364c63f36750dfdf68bf8788b75f Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 14 Jan 2022 15:28:33 -0800 Subject: [PATCH 2/7] Universe: docker, yarn: cue fmt Signed-off-by: Sam Alba --- pkg/universe.dagger.io/docker/build.cue | 1 - .../yarn/test/yarn-test.cue | 74 +++++++++---------- pkg/universe.dagger.io/yarn/yarn.cue | 25 +++---- 3 files changed, 48 insertions(+), 52 deletions(-) diff --git a/pkg/universe.dagger.io/docker/build.cue b/pkg/universe.dagger.io/docker/build.cue index 7ff5c8a8..16e0724b 100644 --- a/pkg/universe.dagger.io/docker/build.cue +++ b/pkg/universe.dagger.io/docker/build.cue @@ -10,7 +10,6 @@ import ( steps: [#Step, ...#Step] output: #Image - // Generate build DAG from linerar steps _dag: { for idx, step in steps { diff --git a/pkg/universe.dagger.io/yarn/test/yarn-test.cue b/pkg/universe.dagger.io/yarn/test/yarn-test.cue index 014f238a..46262d6a 100644 --- a/pkg/universe.dagger.io/yarn/test/yarn-test.cue +++ b/pkg/universe.dagger.io/yarn/test/yarn-test.cue @@ -10,11 +10,9 @@ import ( ) dagger.#Plan & { - inputs: { - directories: { - testdata: path: "./testdata" - testdata2: path: "./testdata2" - } + inputs: directories: { + testdata: path: "./testdata" + testdata2: path: "./testdata2" } actions: { @@ -24,59 +22,59 @@ dagger.#Plan & { } pkg: yarn.#Build & { - source: inputs.directories.testdata.contents + source: inputs.directories.testdata.contents "cache": cache } _image: engine.#Pull & { - source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" - } + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } // FIXME: use bash.#Script test: engine.#Exec & { input: _image.output mounts: build: { - dest: "/build" + dest: "/build" contents: pkg.output } args: [ "sh", "-c", #""" - test "$(cat /build/test)" = "output" - """# + test "$(cat /build/test)" = "output" + """#, ] } } // FIXME: re-enable? // TestSecretsAndFile: { - // pkg: #Package & { - // source: inputs.directories.testdata2 - // writeEnvFile: "/.env" - // env: { - // one: "one" - // two: "two" - // } - // secrets: { - // secretone: dagger.#Secret @dagger(input) - // secretwo: dagger.#Secret @dagger(input) - // } - // } + // pkg: #Package & { + // source: inputs.directories.testdata2 + // writeEnvFile: "/.env" + // env: { + // one: "one" + // two: "two" + // } + // secrets: { + // secretone: dagger.#Secret @dagger(input) + // secretwo: dagger.#Secret @dagger(input) + // } + // } - // test: os.#Container & { - // image: alpine.#Image & { - // package: bash: true - // } - // shell: path: "/bin/bash" - // mount: "/build": from: pkg.build - // command: """ - // content="$(cat /build/env)" - // [[ "${content}" = *"SECRETONE="* ]] && \\ - // [[ "${content}" = *"SECRETWO="* ]] && \\ - // [[ "${content}" = *"ONE=one"* ]] && \\ - // [[ "${content}" = *"TWO=two"* ]] - // """ - // } + // test: os.#Container & { + // image: alpine.#Image & { + // package: bash: true + // } + // shell: path: "/bin/bash" + // mount: "/build": from: pkg.build + // command: """ + // content="$(cat /build/env)" + // [[ "${content}" = *"SECRETONE="* ]] && \\ + // [[ "${content}" = *"SECRETWO="* ]] && \\ + // [[ "${content}" = *"ONE=one"* ]] && \\ + // [[ "${content}" = *"TWO=two"* ]] + // """ + // } // } } -} \ No newline at end of file +} diff --git a/pkg/universe.dagger.io/yarn/yarn.cue b/pkg/universe.dagger.io/yarn/yarn.cue index 4cac4723..ec883468 100644 --- a/pkg/universe.dagger.io/yarn/yarn.cue +++ b/pkg/universe.dagger.io/yarn/yarn.cue @@ -54,29 +54,28 @@ import ( yarnImage: docker.#Run & { image: alpine.image script: """ - apk add -U --no-cache bash yarn - """ + apk add -U --no-cache bash yarn + """ } // Run yarn in a containerized build environment command: bash.#Run & { // FIXME: not working? // *{ - // _image: alpine.#Build & { - // packages: { - // bash: version: "=~5.1" - // yarn: version: yarnVersion - // } - // } + // _image: alpine.#Build & { + // packages: { + // bash: version: "=~5.1" + // yarn: version: yarnVersion + // } + // } - // image: _image.output - // env: CUSTOM_IMAGE: "0" + // image: _image.output + // env: CUSTOM_IMAGE: "0" // } | { - // env: CUSTOM_IMAGE: "1" + // env: CUSTOM_IMAGE: "1" // } - image: docker.#Image & yarnImage.output - + image: docker.#Image & yarnImage.output script: #""" # Create $ENVFILE_NAME file if set From c3f0800c5610035736ee447564227cdf3546a850 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 18 Jan 2022 14:59:51 -0800 Subject: [PATCH 3/7] Universe: yarn: updated docker.#Run syntax Signed-off-by: Sam Alba --- pkg/universe.dagger.io/yarn/yarn.cue | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/universe.dagger.io/yarn/yarn.cue b/pkg/universe.dagger.io/yarn/yarn.cue index ec883468..6b616484 100644 --- a/pkg/universe.dagger.io/yarn/yarn.cue +++ b/pkg/universe.dagger.io/yarn/yarn.cue @@ -100,8 +100,7 @@ import ( // FIXME: mount secrets } - // FIXME - directories: "/build": _ + export: directories: "/build": _ env: { YARN_BUILD_SCRIPT: yarnScript @@ -119,5 +118,5 @@ import ( } // The final contents of the package after build - output: command.directories."/build".contents + output: command.export.directories."/build".contents } From 9650940dd572b427197325b42bff1ed64e7d7b49 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 19 Jan 2022 09:50:06 -0800 Subject: [PATCH 4/7] Universe: yarn: leverage alpine packages for using base image Signed-off-by: Sam Alba --- pkg/universe.dagger.io/yarn/yarn.cue | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/pkg/universe.dagger.io/yarn/yarn.cue b/pkg/universe.dagger.io/yarn/yarn.cue index 6b616484..059b2796 100644 --- a/pkg/universe.dagger.io/yarn/yarn.cue +++ b/pkg/universe.dagger.io/yarn/yarn.cue @@ -7,8 +7,7 @@ import ( "dagger.io/dagger" "dagger.io/dagger/engine" - // "universe.dagger.io/alpine" - "universe.dagger.io/docker" + "universe.dagger.io/alpine" "universe.dagger.io/bash" ) @@ -34,7 +33,7 @@ import ( // Fix for shadowing issues let yarnScript = script - // FIXME: `CacheDir` must be passed by the caller? + // Cache to use, passed by the caller cache: engine.#CacheDir // Optional arguments for the script @@ -44,18 +43,13 @@ import ( // FIXME: not implemented. Are they needed? secrets: [string]: dagger.#Secret + // FIXME: Yarn's version depends on Alpine's version // Yarn version - yarnVersion: *"=~1.22" | string + // yarnVersion: *"=~1.22" | string // FIXME: trouble getting docker.#Build to work (cueflow task dependencies not working) - alpine: docker.#Pull & { - source: "index.docker.io/alpine:3.13.5@sha256:69e70a79f2d41ab5d637de98c1e0b055206ba40a8145e7bddb55ccc04e13cf8f" - } - yarnImage: docker.#Run & { - image: alpine.image - script: """ - apk add -U --no-cache bash yarn - """ + image: alpine.#Build & { + packages: "bash": {} } // Run yarn in a containerized build environment @@ -75,7 +69,7 @@ import ( // env: CUSTOM_IMAGE: "1" // } - image: docker.#Image & yarnImage.output + "image": image.output script: #""" # Create $ENVFILE_NAME file if set From f891065b9f446022cbd06dfdb4f7e07b0a618d6f Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 19 Jan 2022 09:58:38 -0800 Subject: [PATCH 5/7] Universe: fix alpine.#Build Signed-off-by: Sam Alba --- pkg/universe.dagger.io/alpine/alpine.cue | 14 ++++++++------ pkg/universe.dagger.io/yarn/yarn.cue | 10 +++++++--- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/universe.dagger.io/alpine/alpine.cue b/pkg/universe.dagger.io/alpine/alpine.cue index f50fab9b..380a3018 100644 --- a/pkg/universe.dagger.io/alpine/alpine.cue +++ b/pkg/universe.dagger.io/alpine/alpine.cue @@ -22,12 +22,14 @@ let defaultVersion = "3.13.5@sha256:69e70a79f2d41ab5d637de98c1e0b055206ba40a8145 source: "index.docker.io/alpine:\(version)" }, for pkgName, pkg in packages { - run: cmd: { - name: "apk" - args: ["add", "\(pkgName)\(version)"] - flags: { - "-U": true - "--no-cache": true + docker.#Run & { + cmd: { + name: "apk" + args: ["add", "\(pkgName)\(pkg.version)"] + flags: { + "-U": true + "--no-cache": true + } } } }, diff --git a/pkg/universe.dagger.io/yarn/yarn.cue b/pkg/universe.dagger.io/yarn/yarn.cue index 059b2796..05467d78 100644 --- a/pkg/universe.dagger.io/yarn/yarn.cue +++ b/pkg/universe.dagger.io/yarn/yarn.cue @@ -47,9 +47,13 @@ import ( // Yarn version // yarnVersion: *"=~1.22" | string - // FIXME: trouble getting docker.#Build to work (cueflow task dependencies not working) - image: alpine.#Build & { - packages: "bash": {} + image: docker.#Image | *{ + alpine.#Build & { + packages: { + "bash": {} + "yarn": {} + } + } } // Run yarn in a containerized build environment From c887cfa4e326c362ecfa3c39b5ddab2984852c66 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 19 Jan 2022 09:59:06 -0800 Subject: [PATCH 6/7] Universe: yarn: leverage alpine packages for using base image Signed-off-by: Sam Alba --- pkg/universe.dagger.io/yarn/yarn.cue | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/universe.dagger.io/yarn/yarn.cue b/pkg/universe.dagger.io/yarn/yarn.cue index 05467d78..46e34b7e 100644 --- a/pkg/universe.dagger.io/yarn/yarn.cue +++ b/pkg/universe.dagger.io/yarn/yarn.cue @@ -47,14 +47,13 @@ import ( // Yarn version // yarnVersion: *"=~1.22" | string - image: docker.#Image | *{ - alpine.#Build & { + // FIXME: custom base image not supported + image: alpine.#Build & { packages: { - "bash": {} - "yarn": {} + bash: {} + yarn: {} } } - } // Run yarn in a containerized build environment command: bash.#Run & { From 4576b24f2420639241958340ca9d85db005c5cc1 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 19 Jan 2022 11:19:09 -0800 Subject: [PATCH 7/7] engine/types: revert change based from @aluzzardi feedback Signed-off-by: Sam Alba --- pkg/dagger.io/dagger/engine/types.cue | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/dagger.io/dagger/engine/types.cue b/pkg/dagger.io/dagger/engine/types.cue index 15670df0..4c13245c 100644 --- a/pkg/dagger.io/dagger/engine/types.cue +++ b/pkg/dagger.io/dagger/engine/types.cue @@ -7,7 +7,7 @@ package engine // - A directory containing binary artifacts // Rule of thumb: if it fits in a tar archive, it fits in a #FS. #FS: { - $dagger: fs: _id: !="" | null + $dagger: fs: _id: string | null } // A reference to an external secret, for example: @@ -17,7 +17,7 @@ package engine // Secrets are never merged in the Cue tree. They can only be used // by a special filesystem mount designed to minimize leak risk. #Secret: { - $dagger: secret: _id: !="" + $dagger: secret: _id: string } // A reference to a network service endpoint, for example: @@ -25,5 +25,5 @@ package engine // - A unix or npipe socket // - An HTTPS endpoint #Service: { - $dagger: service: _id: !="" + $dagger: service: _id: string }