diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index da719543..6f1b43cb 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -57,7 +57,7 @@ var computeCmd = &cobra.Command{ for _, input := range viper.GetStringSlice("input-dir") { parts := strings.SplitN(input, "=", 2) k, v := parts[0], parts[1] - err := st.SetInput(k, state.DirInput(v, []string{})) + err := st.SetInput(k, state.DirInput(v, []string{}, []string{})) if err != nil { lg. Fatal(). diff --git a/cmd/dagger/cmd/input/dir.go b/cmd/dagger/cmd/input/dir.go index 8d389cdd..4ecbe5c0 100644 --- a/cmd/dagger/cmd/input/dir.go +++ b/cmd/dagger/cmd/input/dir.go @@ -43,11 +43,20 @@ var dirCmd = &cobra.Command{ p = "./" + p } - updateEnvironmentInput(ctx, args[0], state.DirInput(p, []string{})) + updateEnvironmentInput(ctx, args[0], + state.DirInput( + p, + viper.GetStringSlice("include"), + viper.GetStringSlice("exclude"), + ), + ) }, } func init() { + dirCmd.Flags().StringSlice("include", []string{}, "Include pattern") + dirCmd.Flags().StringSlice("exclude", []string{}, "Exclude pattern") + if err := viper.BindPFlags(dirCmd.Flags()); err != nil { panic(err) } diff --git a/environment/environment_test.go b/environment/environment_test.go index f35bc1c4..a2e10973 100644 --- a/environment/environment_test.go +++ b/environment/environment_test.go @@ -12,7 +12,7 @@ func TestLocalDirs(t *testing.T) { Path: "/tmp/source", Plan: "/tmp/source/plan", } - require.NoError(t, st.SetInput("www.source", state.DirInput("/", []string{}))) + require.NoError(t, st.SetInput("www.source", state.DirInput("/", []string{}, []string{}))) environment, err := New(st) require.NoError(t, err) diff --git a/environment/pipeline.go b/environment/pipeline.go index 687371b6..a94de13f 100644 --- a/environment/pipeline.go +++ b/environment/pipeline.go @@ -1,7 +1,6 @@ package environment import ( - "bytes" "context" "encoding/json" "errors" @@ -9,7 +8,6 @@ import ( "io/fs" "net" "net/url" - "path" "strings" "cuelang.org/go/cue" @@ -19,7 +17,6 @@ import ( "github.com/moby/buildkit/exporter/containerimage/exptypes" dockerfilebuilder "github.com/moby/buildkit/frontend/dockerfile/builder" "github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb" - "github.com/moby/buildkit/frontend/dockerfile/dockerignore" bkgw "github.com/moby/buildkit/frontend/gateway/client" bkpb "github.com/moby/buildkit/solver/pb" "github.com/rs/zerolog/log" @@ -29,10 +26,6 @@ import ( "go.dagger.io/dagger/solver" ) -const ( - daggerignoreFilename = ".daggerignore" -) - // An execution pipeline type Pipeline struct { code *compiler.Value @@ -298,71 +291,50 @@ func (p *Pipeline) Local(ctx context.Context, op *compiler.Value, st llb.State) return st, err } - // daggerignore processing - // buildkit related setup - daggerignoreState := llb.Local( - dir, + opts := []llb.LocalOption{ + llb.WithCustomName(p.vertexNamef("Local %s", dir)), + // Without hint, multiple `llb.Local` operations on the + // same path get a different digest. llb.SessionID(p.s.SessionID()), - llb.FollowPaths([]string{daggerignoreFilename}), - llb.SharedKeyHint(dir+"-"+daggerignoreFilename), - llb.WithCustomName(p.vertexNamef("Try loading %s", path.Join(dir, daggerignoreFilename))), - ) - ref, err := p.s.Solve(ctx, daggerignoreState) + llb.SharedKeyHint(dir), + } + + includes, err := op.Lookup("include").List() if err != nil { return st, err } - - // try to read file - var daggerignore []byte - // bool in case file is empty - ignorefound := true - daggerignore, err = ref.ReadFile(ctx, bkgw.ReadRequest{ - Filename: daggerignoreFilename, - }) - // hack for string introspection because !errors.Is(err, os.ErrNotExist) does not work, same for fs - if err != nil { - if !strings.Contains(err.Error(), ".daggerignore: no such file or directory") { - return st, err + if len(includes) > 0 { + includePatterns := []string{} + for _, i := range includes { + pattern, err := i.String() + if err != nil { + return st, err + } + includePatterns = append(includePatterns, pattern) } - ignorefound = false + opts = append(opts, llb.IncludePatterns(includePatterns)) } - // parse out excludes, works even if file does not exist - var excludes []string - excludes, err = dockerignore.ReadAll(bytes.NewBuffer(daggerignore)) + excludes, err := op.Lookup("exclude").List() if err != nil { - return st, fmt.Errorf("%w failed to parse daggerignore", err) + return st, err + } + if len(excludes) > 0 { + excludePatterns := []string{} + for _, i := range excludes { + pattern, err := i.String() + if err != nil { + return st, err + } + excludePatterns = append(excludePatterns, pattern) + } + + opts = append(opts, llb.ExcludePatterns(excludePatterns)) } - // log out patterns if file exists - if ignorefound { - log. - Ctx(ctx). - Debug(). - Str("patterns", fmt.Sprint(excludes)). - Msg("daggerignore exclude patterns") - } - // FIXME: Remove the `Copy` and use `Local` directly. - // - // Copy'ing is a costly operation which should be unnecessary. - // However, using llb.Local directly breaks caching sometimes for unknown reasons. - return st.File( - llb.Copy( - llb.Local( - dir, - // llb.FollowPaths(include), - llb.ExcludePatterns(excludes), - llb.WithCustomName(p.vertexNamef("Local %s [transfer]", dir)), - - // Without hint, multiple `llb.Local` operations on the - // same path get a different digest. - llb.SessionID(p.s.SessionID()), - llb.SharedKeyHint(dir), - ), - "/", - "/", - ), - llb.WithCustomName(p.vertexNamef("Local %s [copy]", dir)), + return llb.Local( + dir, + opts..., ), nil } @@ -811,21 +783,10 @@ func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value, st llb.Stat gitOpts = append(gitOpts, llb.WithCustomName(p.vertexNamef("FetchGit %s@%s", remoteRedacted, ref))) - // FIXME: Remove the `Copy` and use `Git` directly. - // - // Copy'ing is a costly operation which should be unnecessary. - // However, using llb.Git directly breaks caching sometimes for unknown reasons. - return st.File( - llb.Copy( - llb.Git( - remote, - ref, - gitOpts..., - ), - "/", - "/", - ), - llb.WithCustomName(p.vertexNamef("FetchGit %s@%s [copy]", remoteRedacted, ref)), + return llb.Git( + remote, + ref, + gitOpts..., ), nil } diff --git a/state/input.go b/state/input.go index cdf74e45..4506a24f 100644 --- a/state/input.go +++ b/state/input.go @@ -61,18 +61,20 @@ func (i Input) Compile(key string, state *State) (*compiler.Value, error) { } // An input artifact loaded from a local directory -func DirInput(path string, include []string) Input { +func DirInput(path string, include []string, exclude []string) Input { return Input{ Dir: &dirInput{ Path: path, Include: include, + Exclude: exclude, }, } } type dirInput struct { - Path string `json:"path,omitempty"` - Include []string `json:"include,omitempty"` + Path string `yaml:"path,omitempty"` + Include []string `yaml:"include,omitempty"` + Exclude []string `yaml:"exclude,omitempty"` } func (dir dirInput) Compile(_ string, state *State) (*compiler.Value, error) { @@ -88,6 +90,14 @@ func (dir dirInput) Compile(_ string, state *State) (*compiler.Value, error) { return nil, err } } + excludeLLB := []byte("[]") + if len(dir.Exclude) > 0 { + var err error + excludeLLB, err = json.Marshal(dir.Exclude) + if err != nil { + return nil, err + } + } p := dir.Path if !filepath.IsAbs(p) { @@ -98,18 +108,19 @@ func (dir dirInput) Compile(_ string, state *State) (*compiler.Value, error) { } llb := fmt.Sprintf( - `#up: [{do:"local",dir:"%s", include:%s}]`, + `#up: [{do:"local",dir:"%s", include:%s, exclude:%s}]`, p, includeLLB, + excludeLLB, ) return compiler.Compile("", llb) } // An input artifact loaded from a git repository type gitInput struct { - Remote string `json:"remote,omitempty"` - Ref string `json:"ref,omitempty"` - Dir string `json:"dir,omitempty"` + Remote string `yaml:"remote,omitempty"` + Ref string `yaml:"ref,omitempty"` + Dir string `yaml:"dir,omitempty"` } func GitInput(remote, ref, dir string) Input { @@ -145,7 +156,7 @@ func DockerInput(ref string) Input { } type dockerInput struct { - Ref string `json:"ref,omitempty"` + Ref string `yaml:"ref,omitempty"` } func (i dockerInput) Compile(_ string, _ *State) (*compiler.Value, error) { diff --git a/state/state.go b/state/state.go index 84bed925..e00d4f32 100644 --- a/state/state.go +++ b/state/state.go @@ -26,7 +26,7 @@ type State struct { // Cue module containing the environment plan // The input's top-level artifact is used as a module directory. func (s *State) PlanSource() Input { - return DirInput(s.Plan, []string{"*.cue", "cue.mod"}) + return DirInput(s.Plan, []string{"*.cue", "cue.mod"}, []string{}) } func (s *State) SetInput(key string, value Input) error { diff --git a/stdlib/dagger/op/op.cue b/stdlib/dagger/op/op.cue index af8acd5b..252cf910 100644 --- a/stdlib/dagger/op/op.cue +++ b/stdlib/dagger/op/op.cue @@ -36,6 +36,7 @@ package op do: "local" dir: string include: [...string] + exclude: [...string] } // FIXME: bring back load (more efficient than copy) diff --git a/tests/compute.bats b/tests/compute.bats index 97676dd4..c8b1c7aa 100644 --- a/tests/compute.bats +++ b/tests/compute.bats @@ -94,6 +94,6 @@ setup() { assert_output "secret=mySecret" } -@test ".daggerignore" { - "$DAGGER" compute --input-dir TestData="$TESTDIR"/compute/ignore/testdata "$TESTDIR"/compute/ignore +@test "compute: exclude" { + "$DAGGER" up -w "$TESTDIR"/compute/exclude } diff --git a/tests/compute/exclude/.dagger/env/default/.gitignore b/tests/compute/exclude/.dagger/env/default/.gitignore new file mode 100644 index 00000000..01ec19b0 --- /dev/null +++ b/tests/compute/exclude/.dagger/env/default/.gitignore @@ -0,0 +1,2 @@ +# dagger state +state/** diff --git a/tests/compute/ignore/main.cue b/tests/compute/exclude/.dagger/env/default/plan/main.cue similarity index 100% rename from tests/compute/ignore/main.cue rename to tests/compute/exclude/.dagger/env/default/plan/main.cue diff --git a/tests/compute/exclude/.dagger/env/default/values.yaml b/tests/compute/exclude/.dagger/env/default/values.yaml new file mode 100644 index 00000000..c33b528c --- /dev/null +++ b/tests/compute/exclude/.dagger/env/default/values.yaml @@ -0,0 +1,28 @@ +name: default +inputs: + TestData: + dir: + path: ./testdata + exclude: + - a.txt + - '*/*.json' +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArcmRHMFByQkZSdGhteHhm + T2pSWW1UcU5LaFJabFJORllydFk3UkVsSHhVCndQZWNUKzVOeUovTTdCR3FmUXpO + c29GQXhpZkxwSmoweWxqZG1CMkcrRGcKLS0tIDhRMTVSc3NXSWIxSm55TGkwT1E1 + L0NzY0RIMWNkc3k2WStOUGg4SndNRm8Kk7QSP/8spn1Set08VejVW9k4ZwBFqR0T + Ff/N73yNvo633hrfEJtTkhA/aZYyG9bPJy9s9vRDoNFkdTLSFcYX5g== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2021-05-28T23:20:28Z" + mac: ENC[AES256_GCM,data:nuRBGMu6MkiZ5DyuZy3drz4NXoUod8bYzOkJJiIj/fq2shN7oJNShF7UWDpW4FcknD5uldXpKqO3PmGpoxra95TTkoIHsbsQxSIrXUDhXl9CD5WOCwelUstv8f5r4nl+m3tSsW+4rIXdj/9ZB0DulMO0AqPp9I3XHG7glBMWYro=,iv:XGYGZpmC1dOIaTxcEJKtUmv1Fax+8ESPeWnjIGeOVPI=,tag:U/AKlJub9HsMrIpsjDxrYA==,type:str] + pgp: [] + encrypted_suffix: secret + version: 3.7.1 diff --git a/tests/compute/ignore/testdata/a.txt b/tests/compute/exclude/testdata/a.txt similarity index 100% rename from tests/compute/ignore/testdata/a.txt rename to tests/compute/exclude/testdata/a.txt diff --git a/tests/compute/ignore/testdata/b.txt b/tests/compute/exclude/testdata/b.txt similarity index 100% rename from tests/compute/ignore/testdata/b.txt rename to tests/compute/exclude/testdata/b.txt diff --git a/tests/compute/ignore/testdata/foo/bar.txt b/tests/compute/exclude/testdata/foo/bar.txt similarity index 100% rename from tests/compute/ignore/testdata/foo/bar.txt rename to tests/compute/exclude/testdata/foo/bar.txt diff --git a/tests/compute/ignore/testdata/foo/cow.json b/tests/compute/exclude/testdata/foo/cow.json similarity index 100% rename from tests/compute/ignore/testdata/foo/cow.json rename to tests/compute/exclude/testdata/foo/cow.json diff --git a/tests/compute/ignore/testdata/.daggerignore b/tests/compute/ignore/testdata/.daggerignore deleted file mode 100644 index a68c1832..00000000 --- a/tests/compute/ignore/testdata/.daggerignore +++ /dev/null @@ -1,2 +0,0 @@ -a.txt -*/*.json