From 9c81a155c9d258c95827e4968260c26a4fb07c66 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 23 Dec 2021 19:09:26 +0100 Subject: [PATCH 1/2] engine: Make paths relative to the CUE file they're defined in Fixes #1309 Signed-off-by: Andrea Luzzardi --- compiler/value.go | 40 +++++++++++++++++++ plan/plan.go | 11 ++--- plan/task/inputdirectory.go | 16 +++++--- plan/task/inputsecretexec.go | 13 +++++- plan/task/inputsecretfile.go | 20 +++++----- plan/task/outputdirectory.go | 2 +- tests/plan.bats | 22 +++++++++- .../inputs/directories/conflicting_values.cue | 2 +- tests/plan/inputs/directories/exists.cue | 2 +- tests/plan/inputs/secrets/exec_relative.cue | 33 +++++++++++++++ tests/plan/inputs/secrets/test.txt | 1 + tests/tasks/gitpull/private_repo.cue | 2 +- 12 files changed, 134 insertions(+), 30 deletions(-) create mode 100644 tests/plan/inputs/secrets/exec_relative.cue create mode 100644 tests/plan/inputs/secrets/test.txt diff --git a/compiler/value.go b/compiler/value.go index db917504..5e59bf40 100644 --- a/compiler/value.go +++ b/compiler/value.go @@ -1,6 +1,9 @@ package compiler import ( + "errors" + "path" + "path/filepath" "sort" "strconv" @@ -282,6 +285,43 @@ func (v *Value) HasAttr(filter ...string) bool { return false } +// Filename returns the CUE filename where the value was defined +func (v *Value) Filename() (string, error) { + pos := v.Cue().Pos() + if !pos.IsValid() { + return "", errors.New("invalid token position") + } + return pos.Filename(), nil +} + +// Dirname returns the CUE dirname where the value was defined +func (v *Value) Dirname() (string, error) { + f, err := v.Filename() + if err != nil { + return "", err + } + return filepath.Dir(f), nil +} + +// AbsPath returns an absolute path contained in Value +// Paths are relative to the CUE file they were declared in. +func (v *Value) AbsPath() (string, error) { + p, err := v.String() + if err != nil { + return "", nil + } + + if filepath.IsAbs(p) { + return p, nil + } + + d, err := v.Dirname() + if err != nil { + return "", err + } + return path.Join(d, p), nil +} + func (v *Value) Dereference() *Value { dVal := cue.Dereference(v.val) return v.cc.Wrap(dVal) diff --git a/plan/plan.go b/plan/plan.go index 74ad9207..4e4f6e29 100644 --- a/plan/plan.go +++ b/plan/plan.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "os" - "path/filepath" "strings" "time" @@ -97,16 +96,12 @@ func (p *Plan) registerLocalDirs() error { } for _, v := range imports { - dir, err := v.Value.Lookup("path").String() + dir, err := v.Value.Lookup("path").AbsPath() if err != nil { return err } - abs, err := filepath.Abs(dir) - if err != nil { - return err - } - if _, err := os.Stat(abs); errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("path %q does not exist", abs) + if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("path %q does not exist", dir) } p.context.LocalDirs.Add(dir) } diff --git a/plan/task/inputdirectory.go b/plan/task/inputdirectory.go index bf5bfe2b..5e1c074a 100644 --- a/plan/task/inputdirectory.go +++ b/plan/task/inputdirectory.go @@ -18,8 +18,12 @@ type inputDirectoryTask struct { } func (c *inputDirectoryTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + path, err := v.Lookup("path").AbsPath() + if err != nil { + return nil, err + } + var dir struct { - Path string Include []string Exclude []string } @@ -29,13 +33,13 @@ func (c *inputDirectoryTask) Run(ctx context.Context, pctx *plancontext.Context, } lg := log.Ctx(ctx) - lg.Debug().Str("path", dir.Path).Msg("loading local directory") + lg.Debug().Str("path", path).Msg("loading local directory") opts := []llb.LocalOption{ - withCustomName(v, "Local %s", dir.Path), + withCustomName(v, "Local %s", path), // Without hint, multiple `llb.Local` operations on the // same path get a different digest. llb.SessionID(s.SessionID()), - llb.SharedKeyHint(dir.Path), + llb.SharedKeyHint(path), } if len(dir.Include) > 0 { @@ -56,13 +60,13 @@ func (c *inputDirectoryTask) Run(ctx context.Context, pctx *plancontext.Context, st := llb.Scratch().File( llb.Copy( llb.Local( - dir.Path, + path, opts..., ), "/", "/", ), - withCustomName(v, "Local %s [copy]", dir.Path), + withCustomName(v, "Local %s [copy]", path), ) result, err := s.Solve(ctx, st, pctx.Platform.Get()) diff --git a/plan/task/inputsecretexec.go b/plan/task/inputsecretexec.go index 5806cccf..33fb8f63 100644 --- a/plan/task/inputsecretexec.go +++ b/plan/task/inputsecretexec.go @@ -3,6 +3,7 @@ package task import ( "context" "fmt" + "os" "os/exec" "strings" @@ -35,8 +36,18 @@ func (c *inputSecretExecTask) Run(ctx context.Context, pctx *plancontext.Context lg := log.Ctx(ctx) lg.Debug().Str("name", secretExec.Command.Name).Str("args", strings.Join(secretExec.Command.Args, " ")).Str("trimSpace", fmt.Sprintf("%t", secretExec.TrimSpace)).Msg("loading secret") + var err error + + //#nosec G204: sec audited by @aluzzardi and @mrjones + cmd := exec.CommandContext(ctx, secretExec.Command.Name, secretExec.Command.Args...) + cmd.Env = os.Environ() + cmd.Dir, err = v.Lookup("command.name").Dirname() + if err != nil { + return nil, err + } + // sec audited by @aluzzardi and @mrjones - out, err := exec.CommandContext(ctx, secretExec.Command.Name, secretExec.Command.Args...).Output() //#nosec G204 + out, err := cmd.Output() if err != nil { return nil, err } diff --git a/plan/task/inputsecretfile.go b/plan/task/inputsecretfile.go index e985f34f..a8fbe055 100644 --- a/plan/task/inputsecretfile.go +++ b/plan/task/inputsecretfile.go @@ -2,7 +2,6 @@ package task import ( "context" - "fmt" "os" "strings" @@ -20,25 +19,26 @@ type inputSecretFileTask struct { } func (c *inputSecretFileTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { - var secretFile struct { - Path string - TrimSpace bool - } + lg := log.Ctx(ctx) - if err := v.Decode(&secretFile); err != nil { + path, err := v.Lookup("path").AbsPath() + if err != nil { return nil, err } - lg := log.Ctx(ctx) - lg.Debug().Str("path", secretFile.Path).Str("trimSpace", fmt.Sprintf("%t", secretFile.TrimSpace)).Msg("loading secret") + lg.Debug().Str("path", path).Msg("loading secret") - fileBytes, err := os.ReadFile(secretFile.Path) + fileBytes, err := os.ReadFile(path) if err != nil { return nil, err } plaintext := string(fileBytes) - if secretFile.TrimSpace { + trimSpace, err := v.Lookup("trimSpace").Bool() + if err != nil { + return nil, err + } + if trimSpace { plaintext = strings.TrimSpace(plaintext) } diff --git a/plan/task/outputdirectory.go b/plan/task/outputdirectory.go index 82ad41c6..44fbe3d9 100644 --- a/plan/task/outputdirectory.go +++ b/plan/task/outputdirectory.go @@ -22,7 +22,7 @@ func (c outputDirectoryTask) Run(ctx context.Context, pctx *plancontext.Context, return nil, err } - dest, err := v.Lookup("dest").String() + dest, err := v.Lookup("dest").AbsPath() if err != nil { return nil, err } diff --git a/tests/plan.bats b/tests/plan.bats index ef320499..8540ffa6 100644 --- a/tests/plan.bats +++ b/tests/plan.bats @@ -44,11 +44,18 @@ setup() { "$DAGGER" --europa up ./plan/inputs/directories/exists.cue } +@test "plan/inputs/directories relative directories" { + cd "$TESTDIR" + cd "$TESTDIR"/plan/inputs + + "$DAGGER" --europa up ./directories/exists.cue +} + @test "plan/inputs/directories not exists" { cd "$TESTDIR" run "$DAGGER" --europa up ./plan/inputs/directories/not_exists.cue assert_failure - assert_output --partial 'tests/fasdfsdfs" does not exist' + assert_output --partial 'fasdfsdfs" does not exist' } @test "plan/inputs/directories conflicting values" { @@ -63,6 +70,11 @@ setup() { "$DAGGER" --europa up ./plan/inputs/secrets/exec.cue } +@test "plan/inputs/secrets exec relative" { + cd "$TESTDIR" + "$DAGGER" --europa up ./plan/inputs/secrets/exec.cue +} + @test "plan/inputs/secrets invalid command" { cd "$TESTDIR" run "$DAGGER" --europa up ./plan/inputs/secrets/invalid_command.cue @@ -78,6 +90,14 @@ setup() { assert [ -f "./out/test" ] } +@test "plan/outputs relative paths" { + cd "$TESTDIR"/plan + + rm -f "./outputs/out/test" + "$DAGGER" --europa up ./outputs/outputs.cue + assert [ -f "./outputs/out/test" ] +} + @test "plan/platform" { cd "$TESTDIR" diff --git a/tests/plan/inputs/directories/conflicting_values.cue b/tests/plan/inputs/directories/conflicting_values.cue index 69f3694f..60422bea 100644 --- a/tests/plan/inputs/directories/conflicting_values.cue +++ b/tests/plan/inputs/directories/conflicting_values.cue @@ -5,7 +5,7 @@ import ( ) engine.#Plan & { - inputs: directories: test: path: "./plan/inputs/directories" + inputs: directories: test: path: "." actions: verify: engine.#ReadFile & { input: inputs.directories.test.contents path: "test.txt" diff --git a/tests/plan/inputs/directories/exists.cue b/tests/plan/inputs/directories/exists.cue index 76201c34..5f43171a 100644 --- a/tests/plan/inputs/directories/exists.cue +++ b/tests/plan/inputs/directories/exists.cue @@ -5,7 +5,7 @@ import ( ) engine.#Plan & { - inputs: directories: test: path: "./plan/inputs/directories" + inputs: directories: test: path: "." actions: verify: engine.#ReadFile & { input: inputs.directories.test.contents path: "test.txt" diff --git a/tests/plan/inputs/secrets/exec_relative.cue b/tests/plan/inputs/secrets/exec_relative.cue new file mode 100644 index 00000000..271419b1 --- /dev/null +++ b/tests/plan/inputs/secrets/exec_relative.cue @@ -0,0 +1,33 @@ +package main + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + inputs: secrets: echo: command: { + name: "cat" + args: ["./test.txt"] + } + + actions: { + + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + verify: engine.#Exec & { + input: image.output + mounts: secret: { + dest: "/run/secrets/test" + contents: inputs.secrets.echo.contents + } + args: [ + "sh", "-c", + #""" + test "$(cat /run/secrets/test)" = "test" + """#, + ] + } + } +} diff --git a/tests/plan/inputs/secrets/test.txt b/tests/plan/inputs/secrets/test.txt new file mode 100644 index 00000000..30d74d25 --- /dev/null +++ b/tests/plan/inputs/secrets/test.txt @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/tests/tasks/gitpull/private_repo.cue b/tests/tasks/gitpull/private_repo.cue index cce02b5a..d09389e2 100644 --- a/tests/tasks/gitpull/private_repo.cue +++ b/tests/tasks/gitpull/private_repo.cue @@ -5,7 +5,7 @@ import "alpha.dagger.io/europa/dagger/engine" engine.#Plan & { inputs: secrets: token: command: { name: "sops" - args: ["exec-env", "./secrets_sops.yaml", "echo $TestPAT"] + args: ["exec-env", "../../secrets_sops.yaml", "echo $TestPAT"] } actions: { From 1bbc4777af5d8b1036856bf968163578a37e3044 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 23 Dec 2021 13:50:07 -0800 Subject: [PATCH 2/2] tests: fixed secrets_sops.yml path Signed-off-by: Sam Alba --- tests/tasks/build/build_auth.cue | 4 ++-- tests/tasks/pull/pull_auth.cue | 2 +- tests/tasks/push/push.cue | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/tasks/build/build_auth.cue b/tests/tasks/build/build_auth.cue index 2d20fcdc..cb46d0c2 100644 --- a/tests/tasks/build/build_auth.cue +++ b/tests/tasks/build/build_auth.cue @@ -6,10 +6,10 @@ import ( engine.#Plan & { inputs: { - directories: testdata: path: "./tasks/build/testdata" + directories: testdata: path: "./testdata" secrets: dockerHubToken: command: { name: "sops" - args: ["exec-env", "./secrets_sops.yaml", "echo $DOCKERHUB_TOKEN"] + args: ["exec-env", "../../secrets_sops.yaml", "echo $DOCKERHUB_TOKEN"] } } diff --git a/tests/tasks/pull/pull_auth.cue b/tests/tasks/pull/pull_auth.cue index 04515217..64b7697b 100644 --- a/tests/tasks/pull/pull_auth.cue +++ b/tests/tasks/pull/pull_auth.cue @@ -7,7 +7,7 @@ import ( engine.#Plan & { inputs: secrets: dockerHubToken: command: { name: "sops" - args: ["exec-env", "./secrets_sops.yaml", "echo $DOCKERHUB_TOKEN"] + args: ["exec-env", "../../secrets_sops.yaml", "echo $DOCKERHUB_TOKEN"] } actions: pull: engine.#Pull & { source: "daggerio/ci-test:private-pull@sha256:c74f1b1166784193ea6c8f9440263b9be6cae07dfe35e32a5df7a31358ac2060" diff --git a/tests/tasks/push/push.cue b/tests/tasks/push/push.cue index 1ab0410d..2736ef42 100644 --- a/tests/tasks/push/push.cue +++ b/tests/tasks/push/push.cue @@ -8,7 +8,7 @@ import ( engine.#Plan & { inputs: secrets: dockerHubToken: command: { name: "sops" - args: ["exec-env", "./secrets_sops.yaml", "echo $DOCKERHUB_TOKEN"] + args: ["exec-env", "../../secrets_sops.yaml", "echo $DOCKERHUB_TOKEN"] } #auth: [{