Merge pull request #523 from aluzzardi/input-dir-exclude

add support for excludes in input dir

cmd/dagger/cmd/compute.go

@@ -57,7 +57,7 @@ var computeCmd = &cobra.Command{
 		for _, input := range viper.GetStringSlice("input-dir") {
 			parts := strings.SplitN(input, "=", 2)
 			k, v := parts[0], parts[1]
-			err := st.SetInput(k, state.DirInput(v, []string{}))
+			err := st.SetInput(k, state.DirInput(v, []string{}, []string{}))
 			if err != nil {
 				lg.
 					Fatal().

cmd/dagger/cmd/input/dir.go

@@ -43,11 +43,20 @@ var dirCmd = &cobra.Command{
 			p = "./" + p
 		}
 
-		updateEnvironmentInput(ctx, args[0], state.DirInput(p, []string{}))
+		updateEnvironmentInput(ctx, args[0],
+			state.DirInput(
+				p,
+				viper.GetStringSlice("include"),
+				viper.GetStringSlice("exclude"),
+			),
+		)
 	},
 }
 
 func init() {
+	dirCmd.Flags().StringSlice("include", []string{}, "Include pattern")
+	dirCmd.Flags().StringSlice("exclude", []string{}, "Exclude pattern")
+
 	if err := viper.BindPFlags(dirCmd.Flags()); err != nil {
 		panic(err)
 	}

environment/environment_test.go

@@ -12,7 +12,7 @@ func TestLocalDirs(t *testing.T) {
 		Path: "/tmp/source",
 		Plan: "/tmp/source/plan",
 	}
-	require.NoError(t, st.SetInput("www.source", state.DirInput("/", []string{})))
+	require.NoError(t, st.SetInput("www.source", state.DirInput("/", []string{}, []string{})))
 
 	environment, err := New(st)
 	require.NoError(t, err)

environment/pipeline.go

@@ -1,7 +1,6 @@
 package environment
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -9,7 +8,6 @@ import (
 	"io/fs"
 	"net"
 	"net/url"
-	"path"
 	"strings"
 
 	"cuelang.org/go/cue"
@@ -19,7 +17,6 @@ import (
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
 	dockerfilebuilder "github.com/moby/buildkit/frontend/dockerfile/builder"
 	"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb"
-	"github.com/moby/buildkit/frontend/dockerfile/dockerignore"
 	bkgw "github.com/moby/buildkit/frontend/gateway/client"
 	bkpb "github.com/moby/buildkit/solver/pb"
 	"github.com/rs/zerolog/log"
@@ -29,10 +26,6 @@ import (
 	"go.dagger.io/dagger/solver"
 )
 
-const (
-	daggerignoreFilename = ".daggerignore"
-)
-
 // An execution pipeline
 type Pipeline struct {
 	code     *compiler.Value
@@ -298,71 +291,50 @@ func (p *Pipeline) Local(ctx context.Context, op *compiler.Value, st llb.State)
 		return st, err
 	}
 
-	// daggerignore processing
+	opts := []llb.LocalOption{
-	// buildkit related setup
+		llb.WithCustomName(p.vertexNamef("Local %s", dir)),
-	daggerignoreState := llb.Local(
+		// Without hint, multiple `llb.Local` operations on the
-		dir,
+		// same path get a different digest.
 		llb.SessionID(p.s.SessionID()),
-		llb.FollowPaths([]string{daggerignoreFilename}),
+		llb.SharedKeyHint(dir),
-		llb.SharedKeyHint(dir+"-"+daggerignoreFilename),
+	}
-		llb.WithCustomName(p.vertexNamef("Try loading %s", path.Join(dir, daggerignoreFilename))),
+
-	)
+	includes, err := op.Lookup("include").List()
-	ref, err := p.s.Solve(ctx, daggerignoreState)
 	if err != nil {
 		return st, err
 	}
-
+	if len(includes) > 0 {
-	// try to read file
+		includePatterns := []string{}
-	var daggerignore []byte
+		for _, i := range includes {
-	// bool in case file is empty
+			pattern, err := i.String()
-	ignorefound := true
+			if err != nil {
-	daggerignore, err = ref.ReadFile(ctx, bkgw.ReadRequest{
+				return st, err
-		Filename: daggerignoreFilename,
+			}
-	})
+			includePatterns = append(includePatterns, pattern)
-	// hack for string introspection because !errors.Is(err, os.ErrNotExist) does not work, same for fs
-	if err != nil {
-		if !strings.Contains(err.Error(), ".daggerignore: no such file or directory") {
-			return st, err
 		}
-		ignorefound = false
+		opts = append(opts, llb.IncludePatterns(includePatterns))
 	}
 
-	// parse out excludes, works even if file does not exist
+	excludes, err := op.Lookup("exclude").List()
-	var excludes []string
-	excludes, err = dockerignore.ReadAll(bytes.NewBuffer(daggerignore))
 	if err != nil {
-		return st, fmt.Errorf("%w failed to parse daggerignore", err)
+		return st, err
+	}
+	if len(excludes) > 0 {
+		excludePatterns := []string{}
+		for _, i := range excludes {
+			pattern, err := i.String()
+			if err != nil {
+				return st, err
+			}
+			excludePatterns = append(excludePatterns, pattern)
+		}
+
+		opts = append(opts, llb.ExcludePatterns(excludePatterns))
 	}
 
-	// log out patterns if file exists
+	return llb.Local(
-	if ignorefound {
+		dir,
-		log.
+		opts...,
-			Ctx(ctx).
-			Debug().
-			Str("patterns", fmt.Sprint(excludes)).
-			Msg("daggerignore exclude patterns")
-	}
-	// FIXME: Remove the `Copy` and use `Local` directly.
-	//
-	// Copy'ing is a costly operation which should be unnecessary.
-	// However, using llb.Local directly breaks caching sometimes for unknown reasons.
-	return st.File(
-		llb.Copy(
-			llb.Local(
-				dir,
-				// llb.FollowPaths(include),
-				llb.ExcludePatterns(excludes),
-				llb.WithCustomName(p.vertexNamef("Local %s [transfer]", dir)),
-
-				// Without hint, multiple `llb.Local` operations on the
-				// same path get a different digest.
-				llb.SessionID(p.s.SessionID()),
-				llb.SharedKeyHint(dir),
-			),
-			"/",
-			"/",
-		),
-		llb.WithCustomName(p.vertexNamef("Local %s [copy]", dir)),
 	), nil
 }
 
@@ -811,21 +783,10 @@ func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value, st llb.Stat
 
 	gitOpts = append(gitOpts, llb.WithCustomName(p.vertexNamef("FetchGit %s@%s", remoteRedacted, ref)))
 
-	// FIXME: Remove the `Copy` and use `Git` directly.
+	return llb.Git(
-	//
+		remote,
-	// Copy'ing is a costly operation which should be unnecessary.
+		ref,
-	// However, using llb.Git directly breaks caching sometimes for unknown reasons.
+		gitOpts...,
-	return st.File(
-		llb.Copy(
-			llb.Git(
-				remote,
-				ref,
-				gitOpts...,
-			),
-			"/",
-			"/",
-		),
-		llb.WithCustomName(p.vertexNamef("FetchGit %s@%s [copy]", remoteRedacted, ref)),
 	), nil
 }
 

state/input.go

@@ -61,18 +61,20 @@ func (i Input) Compile(key string, state *State) (*compiler.Value, error) {
 }
 
 // An input artifact loaded from a local directory
-func DirInput(path string, include []string) Input {
+func DirInput(path string, include []string, exclude []string) Input {
 	return Input{
 		Dir: &dirInput{
 			Path:    path,
 			Include: include,
+			Exclude: exclude,
 		},
 	}
 }
 
 type dirInput struct {
-	Path    string   `json:"path,omitempty"`
+	Path    string   `yaml:"path,omitempty"`
-	Include []string `json:"include,omitempty"`
+	Include []string `yaml:"include,omitempty"`
+	Exclude []string `yaml:"exclude,omitempty"`
 }
 
 func (dir dirInput) Compile(_ string, state *State) (*compiler.Value, error) {
@@ -88,6 +90,14 @@ func (dir dirInput) Compile(_ string, state *State) (*compiler.Value, error) {
 			return nil, err
 		}
 	}
+	excludeLLB := []byte("[]")
+	if len(dir.Exclude) > 0 {
+		var err error
+		excludeLLB, err = json.Marshal(dir.Exclude)
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	p := dir.Path
 	if !filepath.IsAbs(p) {
@@ -98,18 +108,19 @@ func (dir dirInput) Compile(_ string, state *State) (*compiler.Value, error) {
 	}
 
 	llb := fmt.Sprintf(
-		`#up: [{do:"local",dir:"%s", include:%s}]`,
+		`#up: [{do:"local",dir:"%s", include:%s, exclude:%s}]`,
 		p,
 		includeLLB,
+		excludeLLB,
 	)
 	return compiler.Compile("", llb)
 }
 
 // An input artifact loaded from a git repository
 type gitInput struct {
-	Remote string `json:"remote,omitempty"`
+	Remote string `yaml:"remote,omitempty"`
-	Ref    string `json:"ref,omitempty"`
+	Ref    string `yaml:"ref,omitempty"`
-	Dir    string `json:"dir,omitempty"`
+	Dir    string `yaml:"dir,omitempty"`
 }
 
 func GitInput(remote, ref, dir string) Input {
@@ -145,7 +156,7 @@ func DockerInput(ref string) Input {
 }
 
 type dockerInput struct {
-	Ref string `json:"ref,omitempty"`
+	Ref string `yaml:"ref,omitempty"`
 }
 
 func (i dockerInput) Compile(_ string, _ *State) (*compiler.Value, error) {

state/state.go

@@ -26,7 +26,7 @@ type State struct {
 // Cue module containing the environment plan
 // The input's top-level artifact is used as a module directory.
 func (s *State) PlanSource() Input {
-	return DirInput(s.Plan, []string{"*.cue", "cue.mod"})
+	return DirInput(s.Plan, []string{"*.cue", "cue.mod"}, []string{})
 }
 
 func (s *State) SetInput(key string, value Input) error {

stdlib/dagger/op/op.cue

@@ -36,6 +36,7 @@ package op
 	do:  "local"
 	dir: string
 	include: [...string]
+	exclude: [...string]
 }
 
 // FIXME: bring back load (more efficient than copy)

tests/compute.bats

@@ -94,6 +94,6 @@ setup() {
     assert_output "secret=mySecret"
 }
 
-@test ".daggerignore" {
+@test "compute: exclude" {
-    "$DAGGER" compute --input-dir TestData="$TESTDIR"/compute/ignore/testdata "$TESTDIR"/compute/ignore
+    "$DAGGER" up -w "$TESTDIR"/compute/exclude
 }

tests/compute/exclude/.dagger/env/default/.gitignore

@@ -0,0 +1,2 @@
+# dagger state
+state/**

tests/compute/exclude/.dagger/env/default/values.yaml

@@ -0,0 +1,28 @@
+name: default
+inputs:
+    TestData:
+        dir:
+            path: ./testdata
+            exclude:
+                - a.txt
+                - '*/*.json'
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArcmRHMFByQkZSdGhteHhm
+            T2pSWW1UcU5LaFJabFJORllydFk3UkVsSHhVCndQZWNUKzVOeUovTTdCR3FmUXpO
+            c29GQXhpZkxwSmoweWxqZG1CMkcrRGcKLS0tIDhRMTVSc3NXSWIxSm55TGkwT1E1
+            L0NzY0RIMWNkc3k2WStOUGg4SndNRm8Kk7QSP/8spn1Set08VejVW9k4ZwBFqR0T
+            Ff/N73yNvo633hrfEJtTkhA/aZYyG9bPJy9s9vRDoNFkdTLSFcYX5g==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2021-05-28T23:20:28Z"
+    mac: ENC[AES256_GCM,data:nuRBGMu6MkiZ5DyuZy3drz4NXoUod8bYzOkJJiIj/fq2shN7oJNShF7UWDpW4FcknD5uldXpKqO3PmGpoxra95TTkoIHsbsQxSIrXUDhXl9CD5WOCwelUstv8f5r4nl+m3tSsW+4rIXdj/9ZB0DulMO0AqPp9I3XHG7glBMWYro=,iv:XGYGZpmC1dOIaTxcEJKtUmv1Fax+8ESPeWnjIGeOVPI=,tag:U/AKlJub9HsMrIpsjDxrYA==,type:str]
+    pgp: []
+    encrypted_suffix: secret
+    version: 3.7.1

tests/compute/ignore/testdata/.daggerignore

@@ -1,2 +0,0 @@
-a.txt
-*/*.json