From e9ca8f38e65ec45332a52e493b5532b9f37addc8 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 13 Oct 2021 14:32:06 -0700 Subject: [PATCH] Vendoring improved * update dagger init with package manager downloading stdlib Signed-off-by: Tihomir Jovicic * split mod get and update functions Signed-off-by: Tihomir Jovicic * write to package checksum to dagger.sum when installing/updating Signed-off-by: Tihomir Jovicic * encure checksums are valid when compiling input Signed-off-by: Tihomir Jovicic * remove references to github.com/tjovicic in docs 1010 and 1011 Signed-off-by: Tihomir Jovicic * refactor mod get command Signed-off-by: Tihomir Jovicic * update logic of moving dir when installing packages Signed-off-by: Tihomir Jovicic * fix linting errors Signed-off-by: Tihomir Jovicic * revert changes to 1010 docs Signed-off-by: Tihomir Jovicic * updating error log line in mod/get Signed-off-by: Tihomir Jovicic * fix ci tests when using vendoring Signed-off-by: Tihomir Jovicic * update alpha.dagger.io version to v0.1 Signed-off-by: Tihomir Jovicic * fix mod repo test Signed-off-by: Tihomir Jovicic * return error if package already installed Signed-off-by: Tihomir Jovicic * remove already installed packages when installing Signed-off-by: Tihomir Jovicic * fix issue when vendoring stdlib Signed-off-by: Tihomir Jovicic * update mod command with filelock while installing Signed-off-by: Tihomir Jovicic * fix linting errors Signed-off-by: Tihomir Jovicic * fix path of mod lock file Signed-off-by: Tihomir Jovicic Co-authored-by: Tihomir Jovicic Signed-off-by: Sam Alba --- .gitignore | 4 + cmd/dagger/cmd/mod/arg.go | 57 ---- cmd/dagger/cmd/mod/file.go | 223 ------------- cmd/dagger/cmd/mod/get.go | 80 ++--- docs/learn/1010-dev-cue-package.md | 8 +- docs/learn/1011-package-manager.md | 4 +- go.mod | 2 + go.sum | 4 +- mod/checksum.go | 31 ++ mod/file.go | 293 ++++++++++++++++++ {cmd/dagger/cmd/mod => mod}/file_test.go | 21 +- mod/mod.go | 135 ++++++++ {cmd/dagger/cmd/mod => mod}/repo.go | 30 +- {cmd/dagger/cmd/mod => mod}/repo_test.go | 34 +- mod/require.go | 90 ++++++ .../mod/arg_test.go => mod/require_test.go | 45 ++- .../mod => mod}/test-ssh-keys/id_ed25519_test | 0 .../test-ssh-keys/id_ed25519_test.pub | 0 mod/version.go | 27 ++ state/project.go | 27 +- state/state.go | 21 +- state/state_test.go | 1 + 22 files changed, 709 insertions(+), 428 deletions(-) delete mode 100644 cmd/dagger/cmd/mod/arg.go delete mode 100644 cmd/dagger/cmd/mod/file.go create mode 100644 mod/checksum.go create mode 100644 mod/file.go rename {cmd/dagger/cmd/mod => mod}/file_test.go (64%) create mode 100644 mod/mod.go rename {cmd/dagger/cmd/mod => mod}/repo.go (83%) rename {cmd/dagger/cmd/mod => mod}/repo_test.go (69%) create mode 100644 mod/require.go rename cmd/dagger/cmd/mod/arg_test.go => mod/require_test.go (70%) rename {cmd/dagger/cmd/mod => mod}/test-ssh-keys/id_ed25519_test (100%) rename {cmd/dagger/cmd/mod => mod}/test-ssh-keys/id_ed25519_test.pub (100%) create mode 100644 mod/version.go create mode 100644 state/state_test.go diff --git a/.gitignore b/.gitignore index 8a6a6c9c..43a0d8db 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,7 @@ tests/node_modules dist/ docs/learn/tests/node_modules + +# Integration CI dagger.mod, dagger.sum +dagger.mod +dagger.sum diff --git a/cmd/dagger/cmd/mod/arg.go b/cmd/dagger/cmd/mod/arg.go deleted file mode 100644 index fe372206..00000000 --- a/cmd/dagger/cmd/mod/arg.go +++ /dev/null @@ -1,57 +0,0 @@ -package mod - -import ( - "fmt" - "path" - "regexp" - "strings" -) - -func parseArgument(arg string) (*require, error) { - switch { - case strings.HasPrefix(arg, "github.com"): - return parseGithubRepoName(arg) - case strings.HasPrefix(arg, "alpha.dagger.io"): - return parseDaggerRepoName(arg) - default: - return nil, fmt.Errorf("repo name does not match suported providers") - } -} - -var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`) - -func parseGithubRepoName(arg string) (*require, error) { - repoMatches := githubRepoNameRegex.FindStringSubmatch(arg) - - if len(repoMatches) < 4 { - return nil, fmt.Errorf("issue when parsing github repo") - } - - return &require{ - repo: repoMatches[1], - path: repoMatches[2], - version: repoMatches[3], - - cloneRepo: repoMatches[1], - clonePath: repoMatches[2], - }, nil -} - -var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`) - -func parseDaggerRepoName(arg string) (*require, error) { - repoMatches := daggerRepoNameRegex.FindStringSubmatch(arg) - - if len(repoMatches) < 3 { - return nil, fmt.Errorf("issue when parsing dagger repo") - } - - return &require{ - repo: "alpha.dagger.io", - path: repoMatches[1], - version: repoMatches[2], - - cloneRepo: "github.com/dagger/dagger", - clonePath: path.Join("/stdlib", repoMatches[1]), - }, nil -} diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go deleted file mode 100644 index 4f36b8ef..00000000 --- a/cmd/dagger/cmd/mod/file.go +++ /dev/null @@ -1,223 +0,0 @@ -package mod - -import ( - "bytes" - "errors" - "fmt" - "io" - "io/fs" - "io/ioutil" - "os" - "path" - "path/filepath" - "regexp" - "strings" - - "github.com/spf13/viper" -) - -const filePath = "./cue.mod/dagger.mod" -const destBasePath = "./cue.mod/pkg" -const tmpBasePath = "./cue.mod/tmp" - -// A file is the parsed, interpreted form of a cue.mod file. -type file struct { - require []*require - - projectPath string -} - -func readPath(projectPath string) (*file, error) { - p := path.Join(projectPath, filePath) - - f, err := os.Open(p) - if err != nil { - if !errors.Is(err, fs.ErrNotExist) { - return nil, err - } - - // dagger.mod.cue doesn't exist, let's create an empty file - if f, err = os.Create(p); err != nil { - return nil, err - } - } - - modFile, err := read(f) - if err != nil { - return nil, err - } - - modFile.projectPath = projectPath - - return modFile, nil -} - -func read(f io.Reader) (*file, error) { - b, err := ioutil.ReadAll(f) - if err != nil { - return nil, err - } - - lines := nonEmptyLines(b) - - requires := make([]*require, 0, len(lines)) - for _, line := range lines { - split := strings.Split(line, " ") - r, err := parseArgument(split[0]) - if err != nil { - return nil, err - } - - r.version = split[1] - - requires = append(requires, r) - } - - return &file{ - require: requires, - }, nil -} - -var spaceRgx = regexp.MustCompile(`\s+`) - -func nonEmptyLines(b []byte) []string { - s := strings.ReplaceAll(string(b), "\r\n", "\n") - split := strings.Split(s, "\n") - - lines := make([]string, 0, len(split)) - for _, l := range split { - trimmed := strings.TrimSpace(l) - if trimmed == "" { - continue - } - - trimmed = spaceRgx.ReplaceAllString(trimmed, " ") - - lines = append(lines, trimmed) - } - - return lines -} - -func (f *file) processRequire(req *require, upgrade bool) (bool, error) { - var isNew bool - - tmpPath := path.Join(f.projectPath, tmpBasePath, req.repo) - if err := os.MkdirAll(tmpPath, 0755); err != nil { - return false, fmt.Errorf("error creating tmp dir for cloning package") - } - defer os.RemoveAll(tmpPath) - - // clone the repo - privateKeyFile := viper.GetString("private-key-file") - privateKeyPassword := viper.GetString("private-key-password") - r, err := clone(req, tmpPath, privateKeyFile, privateKeyPassword) - if err != nil { - return isNew, fmt.Errorf("error downloading package %s: %w", req, err) - } - - existing := f.search(req) - destPath := path.Join(f.projectPath, destBasePath) - - // requirement is new, so we should move the files and add it to the mod file - if existing == nil { - if err := move(req, tmpPath, destPath); err != nil { - return isNew, err - } - f.require = append(f.require, req) - isNew = true - return isNew, nil - } - - if upgrade { - latestTag, err := r.latestTag() - if err != nil { - return isNew, err - } - - if latestTag == "" { - return isNew, fmt.Errorf("repo does not have a tag") - } - - req.version = latestTag - } - - c, err := compareVersions(existing.version, req.version) - if err != nil { - return isNew, err - } - - // the existing requirement is newer or equal so we skip installation - if c >= 0 { - return isNew, nil - } - - // the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing - // requirement and replace the code in the /pkg folder - existing.version = req.version - if err = r.checkout(req.version); err != nil { - return isNew, err - } - if err = replace(req, tmpPath, destPath); err != nil { - return isNew, err - } - isNew = true - - return isNew, nil -} - -func (f *file) write() error { - return ioutil.WriteFile(path.Join(f.projectPath, filePath), f.contents().Bytes(), 0600) -} - -func (f *file) contents() *bytes.Buffer { - var b bytes.Buffer - - for _, r := range f.require { - b.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.version)) - } - - return &b -} - -func (f *file) search(r *require) *require { - for _, existing := range f.require { - if existing.fullPath() == r.fullPath() { - return existing - } - } - return nil -} - -type require struct { - repo string - path string - version string - - cloneRepo string - clonePath string -} - -func (r *require) fullPath() string { - return path.Join(r.repo, r.path) -} - -func move(r *require, sourceRepoPath, destBasePath string) error { - destPath := path.Join(destBasePath, r.fullPath()) - if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { - return err - } - if err := os.Rename(path.Join(sourceRepoPath, r.path), destPath); err != nil { - return err - } - - return nil -} - -func replace(r *require, sourceRepoPath, destBasePath string) error { - if err := os.RemoveAll(path.Join(destBasePath, r.fullPath())); err != nil { - return err - } - - return move(r, sourceRepoPath, destBasePath) -} diff --git a/cmd/dagger/cmd/mod/get.go b/cmd/dagger/cmd/mod/get.go index 23b1e8a4..c4cca65b 100644 --- a/cmd/dagger/cmd/mod/get.go +++ b/cmd/dagger/cmd/mod/get.go @@ -1,11 +1,11 @@ package mod import ( - "github.com/hashicorp/go-version" "github.com/spf13/cobra" "github.com/spf13/viper" "go.dagger.io/dagger/cmd/dagger/cmd/common" "go.dagger.io/dagger/cmd/dagger/logger" + "go.dagger.io/dagger/mod" "go.dagger.io/dagger/telemetry" ) @@ -32,77 +32,41 @@ var getCmd = &cobra.Command{ Value: args, }) - // read mod file in the current dir - modFile, err := readPath(project.Path) - if err != nil { - lg.Fatal().Err(err).Msg("error loading module file") - } + var update = viper.GetBool("update") - // parse packages to install - var packages []*require - var upgrade bool - - if len(args) == 0 { - lg.Info().Msg("upgrading installed packages...") - packages = modFile.require - upgrade = true + var processedRequires []*mod.Require + var err error + if update && len(args) == 0 { + lg.Info().Msg("updating all installed packages...") + processedRequires, err = mod.UpdateInstalled(project.Path) + } else if update && len(args) > 0 { + lg.Info().Msg("updating specified packages...") + processedRequires, err = mod.UpdateAll(project.Path, args) + } else if !update && len(args) > 0 { + lg.Info().Msg("installing specified packages...") + processedRequires, err = mod.InstallAll(project.Path, args) } else { - for _, arg := range args { - p, err := parseArgument(arg) - if err != nil { - lg.Error().Err(err).Msgf("error parsing package %s", arg) - continue - } - packages = append(packages, p) + lg.Fatal().Msg("unrecognized update/install operation") + } + + if len(processedRequires) > 0 { + for _, r := range processedRequires { + lg.Info().Msgf("installed/updated package %s", r) } } - // download packages - for _, p := range packages { - isNew, err := modFile.processRequire(p, upgrade) - if err != nil { - lg.Error().Err(err).Msgf("error processing package %s", p.repo) - } - - if isNew { - lg.Info().Msgf("downloading %s:%v", p.repo, p.version) - } - } - - // write to mod file in the current dir - if err = modFile.write(); err != nil { - lg.Error().Err(err).Msg("error writing to mod file") + if err != nil { + lg.Error().Err(err).Msg("error installing/updating packages") } <-doneCh }, } -func compareVersions(reqV1, reqV2 string) (int, error) { - v1, err := version.NewVersion(reqV1) - if err != nil { - return 0, err - } - - v2, err := version.NewVersion(reqV2) - if err != nil { - return 0, err - } - - if v1.LessThan(v2) { - return -1, nil - } - - if v1.Equal(v2) { - return 0, nil - } - - return 1, nil -} - func init() { getCmd.Flags().String("private-key-file", "", "Private ssh key") getCmd.Flags().String("private-key-password", "", "Private ssh key password") + getCmd.Flags().BoolP("update", "u", false, "Update specified package") if err := viper.BindPFlags(getCmd.Flags()); err != nil { panic(err) diff --git a/docs/learn/1010-dev-cue-package.md b/docs/learn/1010-dev-cue-package.md index c44071a0..11c329c4 100644 --- a/docs/learn/1010-dev-cue-package.md +++ b/docs/learn/1010-dev-cue-package.md @@ -42,13 +42,13 @@ domain name (as in Go) followed by a descriptive name. In this example, we reuse package from it. ```shell -mkdir -p cue.mod/pkg/github.com/tjovicic/gcpcloudrun +mkdir -p cue.mod/pkg/github.com/username/gcpcloudrun ``` Let's write the package logic. It is basically what we've seen in the 106-cloudrun example: ```shell -touch cue.mod/pkg/github.com/tjovicic/gcpcloudrun/source.cue +touch cue.mod/pkg/github.com/username/gcpcloudrun/source.cue ``` ```cue file=./tests/dev-cue-package/source.cue title="cue.mod/pkg/github.com/tjovicic/gcpcloudrun/source.cue" @@ -86,8 +86,8 @@ You've probably guessed this package isn't tied to just your project. You can ea of different projects and use it as we've shown above. ```shell -mkdir -p /my-new-project/cue.mod/pkg/github.com/tjovicic/gcpcloudrun -cp ./cue.mod/pkg/github.com/tjovicic/gcpcloudrun/source.cue /new-project/cue.mod/pkg/github.com/tjovicic/gcpcloudrun +mkdir -p /my-new-workspace/cue.mod/pkg/github.com/username/gcpcloudrun +cp ./cue.mod/pkg/github.com/username/gcpcloudrun/source.cue /new-workspace/cue.mod/pkg/github.com/username/gcpcloudrun ``` ## Contributing to Dagger stdlib diff --git a/docs/learn/1011-package-manager.md b/docs/learn/1011-package-manager.md index 5f01d8cf..24394813 100644 --- a/docs/learn/1011-package-manager.md +++ b/docs/learn/1011-package-manager.md @@ -59,7 +59,7 @@ dagger mod get github.com/dagger/packages/gcpcloudrun@v0.1 ``` It should pull the `v0.1` version from Github, leave a copy in `cue.mod/pkg` and reflect the change in -`cue.mod/dagger.mod.cue` file: +`cue.mod/dagger.mod` file: ```shell cue.mod/pkg/github.com/ @@ -111,7 +111,7 @@ You should see similar output: 12:25PM INF system | downloading github.com/dagger/packages:v0.2 ``` -And `cue.mod/dagger.mod.cue` should reflect the new version: +And `cue.mod/dagger.mod` should reflect the new version: ```cue title="./cue.mod/dagger.mod" github.com/dagger/packages/gcpcloudrun v0.2 diff --git a/go.mod b/go.mod index e7ea64c5..ae1c7b69 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/docker/distribution v2.7.1+incompatible github.com/emicklei/proto v1.9.0 // indirect github.com/go-git/go-git/v5 v5.4.2 + github.com/gofrs/flock v0.8.1 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/google/uuid v1.3.0 github.com/hashicorp/go-version v1.3.0 @@ -34,6 +35,7 @@ require ( go.opentelemetry.io/otel/sdk v1.0.1 go.opentelemetry.io/otel/trace v1.0.1 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/mod v0.4.2 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef // indirect golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b diff --git a/go.sum b/go.sum index 2a5ea62d..a0b497c3 100644 --- a/go.sum +++ b/go.sum @@ -594,8 +594,9 @@ github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gofrs/flock v0.7.3 h1:I0EKY9l8HZCXTMYC4F80vwT6KNypV9uYKP3Alm/hjmQ= github.com/gofrs/flock v0.7.3/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.3.2/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= @@ -1503,6 +1504,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/mod/checksum.go b/mod/checksum.go new file mode 100644 index 00000000..a9637356 --- /dev/null +++ b/mod/checksum.go @@ -0,0 +1,31 @@ +package mod + +import ( + "fmt" + "os" + "path" + + "golang.org/x/mod/sumdb/dirhash" +) + +func dirChecksum(dirPath string) (string, error) { + err := cleanDirForChecksum(dirPath) + if err != nil { + return "", err + } + + checksum, err := dirhash.HashDir(dirPath, "", dirhash.DefaultHash) + if err != nil { + return "", err + } + + return checksum, nil +} + +func cleanDirForChecksum(dirPath string) error { + if err := os.RemoveAll(path.Join(dirPath, ".git")); err != nil { + return fmt.Errorf("error cleaning up .git directory") + } + + return nil +} diff --git a/mod/file.go b/mod/file.go new file mode 100644 index 00000000..947ea583 --- /dev/null +++ b/mod/file.go @@ -0,0 +1,293 @@ +package mod + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "io/ioutil" + "os" + "path" + "regexp" + "strings" + + "github.com/spf13/viper" +) + +const modFilePath = "./cue.mod/dagger.mod" +const sumFilePath = "./cue.mod/dagger.sum" +const lockFilePath = "./cue.mod/dagger.lock" +const destBasePath = "./cue.mod/pkg" +const tmpBasePath = "./cue.mod/tmp" + +// file is the parsed, interpreted form of dagger.mod file. +type file struct { + requires []*Require + workspacePath string +} + +func readPath(workspacePath string) (*file, error) { + pMod := path.Join(workspacePath, modFilePath) + fMod, err := os.Open(pMod) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + + // dagger.mod doesn't exist, let's create an empty file + if fMod, err = os.Create(pMod); err != nil { + return nil, err + } + } + + pSum := path.Join(workspacePath, sumFilePath) + fSum, err := os.Open(pSum) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + + // dagger.sum doesn't exist, let's create an empty file + if fSum, err = os.Create(pSum); err != nil { + return nil, err + } + } + + modFile, err := read(fMod, fSum) + if err != nil { + return nil, err + } + + modFile.workspacePath = workspacePath + + return modFile, nil +} + +func read(fMod, fSum io.Reader) (*file, error) { + bMod, err := ioutil.ReadAll(fMod) + if err != nil { + return nil, err + } + + bSum, err := ioutil.ReadAll(fSum) + if err != nil { + return nil, err + } + + modLines := nonEmptyLines(bMod) + sumLines := nonEmptyLines(bSum) + + if len(modLines) != len(sumLines) { + return nil, fmt.Errorf("length of dagger.mod and dagger.sum files differ") + } + + requires := make([]*Require, 0, len(modLines)) + for i := 0; i < len(modLines); i++ { + modSplit := strings.Split(modLines[i], " ") + if len(modSplit) != 2 { + return nil, fmt.Errorf("line in the mod file doesn't contain 2 elements") + } + + sumSplit := strings.Split(sumLines[i], " ") + if len(sumSplit) != 2 { + return nil, fmt.Errorf("line in the sum file doesn't contain 2 elements") + } + + if modSplit[0] != sumSplit[0] { + return nil, fmt.Errorf("repos in mod and sum line don't match: %s and %s", modSplit[0], sumSplit[0]) + } + + require, err := newRequire(modSplit[0]) + if err != nil { + return nil, err + } + + require.version = modSplit[1] + require.checksum = sumSplit[1] + + requires = append(requires, require) + } + + return &file{requires: requires}, nil +} + +var spaceRgx = regexp.MustCompile(`\s+`) + +func nonEmptyLines(b []byte) []string { + s := strings.ReplaceAll(string(b), "\r\n", "\n") + split := strings.Split(s, "\n") + + lines := make([]string, 0, len(split)) + for _, l := range split { + trimmed := strings.TrimSpace(l) + if trimmed == "" { + continue + } + + trimmed = spaceRgx.ReplaceAllString(trimmed, " ") + + lines = append(lines, trimmed) + } + + return lines +} + +func (f *file) install(req *Require) error { + // cleaning up possible leftovers + tmpPath := path.Join(f.workspacePath, tmpBasePath, req.fullPath()) + defer os.RemoveAll(tmpPath) + + // clone to a tmp directory + r, err := clone(req, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password")) + if err != nil { + return fmt.Errorf("error downloading package %s: %w", req, err) + } + + destPath := path.Join(f.workspacePath, destBasePath, req.fullPath()) + + // requirement is new, so we should move the cloned files from tmp to pkg and add it to the mod file + existing := f.searchInstalledRequire(req) + if existing == nil { + if err = replace(req, tmpPath, destPath); err != nil { + return err + } + + checksum, err := dirChecksum(destPath) + if err != nil { + return err + } + + req.checksum = checksum + + f.requires = append(f.requires, req) + + return nil + } + + // checkout the cloned repo to that tag, change the version in the existing requirement and + // replace the code in the /pkg folder + existing.version = req.version + if err = r.checkout(req.version); err != nil { + return err + } + + if err = replace(req, tmpPath, destPath); err != nil { + return err + } + + checksum, err := dirChecksum(destPath) + if err != nil { + return err + } + + existing.checksum = checksum + + return nil +} + +func (f *file) updateToLatest(req *Require) (*Require, error) { + // check if it doesn't exist + existing := f.searchInstalledRequire(req) + if existing == nil { + return nil, fmt.Errorf("package %s isn't already installed", req.fullPath()) + } + + // cleaning up possible leftovers + tmpPath := path.Join(f.workspacePath, tmpBasePath, existing.fullPath()) + defer os.RemoveAll(tmpPath) + + // clone to a tmp directory + gitRepo, err := clone(existing, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password")) + if err != nil { + return nil, fmt.Errorf("error downloading package %s: %w", existing, err) + } + + // checkout the latest tag + latestTag, err := gitRepo.latestTag() + if err != nil { + return nil, err + } + + c, err := compareVersions(latestTag, existing.version) + if err != nil { + return nil, err + } + + if c < 0 { + return nil, fmt.Errorf("latest git tag is less than the current version") + } + + existing.version = latestTag + if err = gitRepo.checkout(existing.version); err != nil { + return nil, err + } + + // move the package from tmp to pkg directory + destPath := path.Join(f.workspacePath, destBasePath, existing.fullPath()) + if err = replace(existing, tmpPath, destPath); err != nil { + return nil, err + } + + checksum, err := dirChecksum(destPath) + if err != nil { + return nil, err + } + + req.checksum = checksum + + return existing, nil +} + +func (f *file) searchInstalledRequire(r *Require) *Require { + for _, existing := range f.requires { + if existing.fullPath() == r.fullPath() { + return existing + } + } + + return nil +} + +func (f *file) ensure() error { + for _, require := range f.requires { + requirePath := path.Join(f.workspacePath, destBasePath, require.fullPath()) + + checksum, err := dirChecksum(requirePath) + if err != nil { + return nil + } + + if require.checksum != checksum { + return fmt.Errorf("wrong checksum for %s", require.fullPath()) + } + } + + return nil +} + +func (f *file) write() error { + // write dagger.mod file + var bMod bytes.Buffer + for _, r := range f.requires { + bMod.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.version)) + } + + err := ioutil.WriteFile(path.Join(f.workspacePath, modFilePath), bMod.Bytes(), 0600) + if err != nil { + return err + } + + // write dagger.sum file + var bSum bytes.Buffer + for _, r := range f.requires { + bSum.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.checksum)) + } + + err = ioutil.WriteFile(path.Join(f.workspacePath, sumFilePath), bSum.Bytes(), 0600) + if err != nil { + return err + } + + return nil +} diff --git a/cmd/dagger/cmd/mod/file_test.go b/mod/file_test.go similarity index 64% rename from cmd/dagger/cmd/mod/file_test.go rename to mod/file_test.go index 1b1c5a4d..e0d5f40b 100644 --- a/cmd/dagger/cmd/mod/file_test.go +++ b/mod/file_test.go @@ -7,18 +7,23 @@ import ( func TestReadFile(t *testing.T) { cases := []struct { - name string - input string - want *file + name string + modFile string + sumFile string + want *file }{ { name: "module file with valid dependencies", - input: ` + modFile: ` github.com/tjovicic/test xyz github.com/bla/bla abc `, + sumFile: ` + github.com/tjovicic/test h1:hash + github.com/bla/bla h1:hash + `, want: &file{ - require: []*require{ + requires: []*Require{ { repo: "github.com/tjovicic/test", path: "", @@ -36,13 +41,13 @@ func TestReadFile(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - got, err := read(strings.NewReader(c.input)) + got, err := read(strings.NewReader(c.modFile), strings.NewReader(c.sumFile)) if err != nil { t.Error(err) } - if len(got.require) != len(c.want.require) { - t.Errorf("requires length differs: want %d, got %d", len(c.want.require), len(got.require)) + if len(got.requires) != len(c.want.requires) { + t.Errorf("requires length differs: want %d, got %d", len(c.want.requires), len(got.requires)) } }) } diff --git a/mod/mod.go b/mod/mod.go new file mode 100644 index 00000000..6e7902db --- /dev/null +++ b/mod/mod.go @@ -0,0 +1,135 @@ +package mod + +import ( + "path" + + "github.com/gofrs/flock" +) + +func InstallStdlib(workspace string) error { + _, err := Install(workspace, "alpha.dagger.io@v0.1") + + return err +} + +func Install(workspace, repoName string) (*Require, error) { + require, err := newRequire(repoName) + if err != nil { + return nil, err + } + + modfile, err := readPath(workspace) + if err != nil { + return nil, err + } + + fileLock := flock.New(path.Join(workspace, lockFilePath)) + if err := fileLock.Lock(); err != nil { + return nil, err + } + + err = modfile.install(require) + if err != nil { + return nil, err + } + + if err = modfile.write(); err != nil { + return nil, err + } + + if err := fileLock.Unlock(); err != nil { + return nil, err + } + + return require, nil +} + +func InstallAll(workspace string, repoNames []string) ([]*Require, error) { + installedRequires := make([]*Require, 0, len(repoNames)) + var err error + + for _, repoName := range repoNames { + var require *Require + + if require, err = Install(workspace, repoName); err != nil { + continue + } + + installedRequires = append(installedRequires, require) + } + + return installedRequires, err +} + +func Update(workspace, repoName string) (*Require, error) { + require, err := newRequire(repoName) + if err != nil { + return nil, err + } + + modfile, err := readPath(workspace) + if err != nil { + return nil, err + } + + fileLock := flock.New(path.Join(workspace, lockFilePath)) + if err := fileLock.Lock(); err != nil { + return nil, err + } + + updatedRequire, err := modfile.updateToLatest(require) + if err != nil { + return nil, err + } + + if err = modfile.write(); err != nil { + return nil, err + } + + if err := fileLock.Unlock(); err != nil { + return nil, err + } + + return updatedRequire, nil +} + +func UpdateAll(workspace string, repoNames []string) ([]*Require, error) { + updatedRequires := make([]*Require, 0, len(repoNames)) + var err error + + for _, repoName := range repoNames { + var require *Require + + if require, err = Update(workspace, repoName); err != nil { + continue + } + + updatedRequires = append(updatedRequires, require) + } + + return updatedRequires, err +} + +func UpdateInstalled(workspace string) ([]*Require, error) { + modfile, err := readPath(workspace) + if err != nil { + return nil, err + } + + repoNames := make([]string, 0, len(modfile.requires)) + + for _, require := range modfile.requires { + repoNames = append(repoNames, require.String()) + } + + return UpdateAll(workspace, repoNames) +} + +func Ensure(workspace string) error { + modfile, err := readPath(workspace) + if err != nil { + return err + } + + return modfile.ensure() +} diff --git a/cmd/dagger/cmd/mod/repo.go b/mod/repo.go similarity index 83% rename from cmd/dagger/cmd/mod/repo.go rename to mod/repo.go index a5b7f3f2..96182a64 100644 --- a/cmd/dagger/cmd/mod/repo.go +++ b/mod/repo.go @@ -3,7 +3,6 @@ package mod import ( "fmt" "os" - "path" "sort" "strings" @@ -15,11 +14,17 @@ import ( ) type repo struct { - localPath string - contents *git.Repository + contents *git.Repository } -func clone(require *require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) { +func clone(require *Require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) { + if err := os.RemoveAll(dir); err != nil { + return nil, fmt.Errorf("error cleaning up tmp directory") + } + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("error creating tmp dir for cloning package") + } + o := git.CloneOptions{ URL: fmt.Sprintf("https://%s", require.cloneRepo), } @@ -40,8 +45,7 @@ func clone(require *require, dir string, privateKeyFile, privateKeyPassword stri } rr := &repo{ - localPath: dir, - contents: r, + contents: r, } if require.version == "" { @@ -50,10 +54,6 @@ func clone(require *require, dir string, privateKeyFile, privateKeyPassword stri return nil, err } - if latestTag == "" { - return nil, fmt.Errorf("no git tags found in the repo") - } - require.version = latestTag } @@ -61,10 +61,6 @@ func clone(require *require, dir string, privateKeyFile, privateKeyPassword stri return nil, err } - if _, err := os.Stat(path.Join(dir, require.clonePath, filePath)); err != nil { - return nil, fmt.Errorf("repo does not contain %s", filePath) - } - return rr, nil } @@ -118,11 +114,11 @@ func (r *repo) latestTag() (string, error) { versions[i] = v } - sort.Sort(version.Collection(versions)) - if len(versions) == 0 { - return "", nil + return "", fmt.Errorf("repo doesn't have any tags") } + sort.Sort(version.Collection(versions)) + return versions[len(versions)-1].Original(), nil } diff --git a/cmd/dagger/cmd/mod/repo_test.go b/mod/repo_test.go similarity index 69% rename from cmd/dagger/cmd/mod/repo_test.go rename to mod/repo_test.go index 19375d4b..e911b581 100644 --- a/cmd/dagger/cmd/mod/repo_test.go +++ b/mod/repo_test.go @@ -9,37 +9,37 @@ import ( func TestClone(t *testing.T) { cases := []struct { name string - require require + require Require privateKeyFile string privateKeyPassword string }{ { name: "resolving shorter hash version", - require: require{ - cloneRepo: "github.com/tjovicic/dagger-modules", - clonePath: "gcpcloudrun", - version: "26a1d46d1b3c", + require: Require{ + cloneRepo: "github.com/dagger/universe", + clonePath: "stdlib", + version: "24d7af3fc2a3e9c7cc2", }, }, { name: "resolving branch name", - require: require{ - cloneRepo: "github.com/tjovicic/dagger-modules", - clonePath: "gcpcloudrun", + require: Require{ + cloneRepo: "github.com/dagger/universe", + clonePath: "stdlib", version: "main", }, }, { name: "resolving tag", - require: require{ - cloneRepo: "github.com/tjovicic/dagger-modules", - clonePath: "gcpcloudrun", - version: "v0.3", + require: Require{ + cloneRepo: "github.com/dagger/universe", + clonePath: "stdlib", + version: "v0.1", }, }, { - name: "Dagger private test repo", - require: require{ + name: "dagger private repo", + require: Require{ cloneRepo: "github.com/dagger/test", clonePath: "", version: "main", @@ -73,9 +73,9 @@ func TestListTags(t *testing.T) { } defer os.Remove(tmpDir) - r, err := clone(&require{ - cloneRepo: "github.com/tjovicic/dagger-modules", - clonePath: "gcpcloudrun", + r, err := clone(&Require{ + cloneRepo: "github.com/dagger/universe", + clonePath: "stdlib", version: "", }, tmpDir, "", "") if err != nil { diff --git a/mod/require.go b/mod/require.go new file mode 100644 index 00000000..fb113692 --- /dev/null +++ b/mod/require.go @@ -0,0 +1,90 @@ +package mod + +import ( + "fmt" + "os" + "path" + "regexp" + "strings" +) + +type Require struct { + repo string + path string + + cloneRepo string + clonePath string + + version string + checksum string +} + +func newRequire(repoName string) (*Require, error) { + switch { + case strings.HasPrefix(repoName, "github.com"): + return parseGithubRepoName(repoName) + case strings.HasPrefix(repoName, "alpha.dagger.io"): + return parseDaggerRepoName(repoName) + default: + return nil, fmt.Errorf("repo name does not match suported providers") + } +} + +var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`) + +func parseGithubRepoName(repoName string) (*Require, error) { + repoMatches := githubRepoNameRegex.FindStringSubmatch(repoName) + + if len(repoMatches) < 4 { + return nil, fmt.Errorf("issue when parsing github repo") + } + + return &Require{ + repo: repoMatches[1], + path: repoMatches[2], + version: repoMatches[3], + + cloneRepo: repoMatches[1], + clonePath: repoMatches[2], + }, nil +} + +var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`) + +func parseDaggerRepoName(repoName string) (*Require, error) { + repoMatches := daggerRepoNameRegex.FindStringSubmatch(repoName) + + if len(repoMatches) < 3 { + return nil, fmt.Errorf("issue when parsing dagger repo") + } + + return &Require{ + repo: "alpha.dagger.io", + path: repoMatches[1], + version: repoMatches[2], + + cloneRepo: "github.com/dagger/universe", + clonePath: path.Join("/stdlib", repoMatches[1]), + }, nil +} + +func (r *Require) String() string { + return fmt.Sprintf("%s@%s", r.fullPath(), r.version) +} + +func (r *Require) fullPath() string { + return path.Join(r.repo, r.path) +} + +func replace(r *Require, sourceRepoPath, destPath string) error { + // remove previous package directory + if err := os.RemoveAll(destPath); err != nil { + return err + } + + if err := os.Rename(path.Join(sourceRepoPath, r.clonePath), destPath); err != nil { + return err + } + + return nil +} diff --git a/cmd/dagger/cmd/mod/arg_test.go b/mod/require_test.go similarity index 70% rename from cmd/dagger/cmd/mod/arg_test.go rename to mod/require_test.go index 4aa1d3fc..e455456e 100644 --- a/cmd/dagger/cmd/mod/arg_test.go +++ b/mod/require_test.go @@ -1,12 +1,14 @@ package mod -import "testing" +import ( + "testing" +) func TestParseArgument(t *testing.T) { cases := []struct { name string in string - want *require + want *Require hasError bool }{ { @@ -17,7 +19,7 @@ func TestParseArgument(t *testing.T) { { name: "Dagger repo", in: "github.com/dagger/dagger", - want: &require{ + want: &Require{ repo: "github.com/dagger/dagger", path: "", version: "", @@ -26,7 +28,7 @@ func TestParseArgument(t *testing.T) { { name: "Dagger repo with path", in: "github.com/dagger/dagger/stdlib", - want: &require{ + want: &Require{ repo: "github.com/dagger/dagger", path: "/stdlib", version: "", @@ -35,7 +37,7 @@ func TestParseArgument(t *testing.T) { { name: "Dagger repo with longer path", in: "github.com/dagger/dagger/stdlib/test/test", - want: &require{ + want: &Require{ repo: "github.com/dagger/dagger", path: "/stdlib/test/test", version: "", @@ -44,25 +46,25 @@ func TestParseArgument(t *testing.T) { { name: "Dagger repo with path and version", in: "github.com/dagger/dagger/stdlib@v0.1", - want: &require{ + want: &Require{ repo: "github.com/dagger/dagger", path: "/stdlib", version: "v0.1", }, }, { - name: "Dagger repo with longer path and version", + name: "Dagger repo with longer path and version tag", in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1", - want: &require{ + want: &Require{ repo: "github.com/dagger/dagger", path: "/stdlib/test/test", version: "v0.0.1", }, }, { - name: "Alpha Dagger repo", + name: "Alpha Dagger repo with path", in: "alpha.dagger.io/gcp/gke@v0.1.0-alpha.20", - want: &require{ + want: &Require{ repo: "alpha.dagger.io", path: "/gcp/gke", version: "v0.1.0-alpha.20", @@ -71,11 +73,32 @@ func TestParseArgument(t *testing.T) { clonePath: "/stdlib/gcp/gke", }, }, + { + name: "Alpha Dagger repo", + in: "alpha.dagger.io@v0.1.0-alpha.23", + want: &Require{ + repo: "alpha.dagger.io", + path: "", + version: "v0.1.0-alpha.23", + + cloneRepo: "github.com/dagger/dagger", + clonePath: "/stdlib", + }, + }, + { + name: "Dagger repo with longer path and commit version", + in: "github.com/dagger/dagger/stdlib/test/test@26a1d46d1b3c", + want: &Require{ + repo: "github.com/dagger/dagger", + path: "/stdlib/test/test", + version: "26a1d46d1b3c", + }, + }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - got, err := parseArgument(c.in) + got, err := newRequire(c.in) if err != nil && c.hasError { return } diff --git a/cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test b/mod/test-ssh-keys/id_ed25519_test similarity index 100% rename from cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test rename to mod/test-ssh-keys/id_ed25519_test diff --git a/cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test.pub b/mod/test-ssh-keys/id_ed25519_test.pub similarity index 100% rename from cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test.pub rename to mod/test-ssh-keys/id_ed25519_test.pub diff --git a/mod/version.go b/mod/version.go new file mode 100644 index 00000000..ebefec6b --- /dev/null +++ b/mod/version.go @@ -0,0 +1,27 @@ +package mod + +import "github.com/hashicorp/go-version" + +// compareVersions returns -1 if the first argument is less or 1 if it's greater than the second argument. +// It returns 0 if they are equal. +func compareVersions(reqV1, reqV2 string) (int, error) { + v1, err := version.NewVersion(reqV1) + if err != nil { + return 0, err + } + + v2, err := version.NewVersion(reqV2) + if err != nil { + return 0, err + } + + if v1.LessThan(v2) { + return -1, nil + } + + if v1.Equal(v2) { + return 0, nil + } + + return 1, nil +} diff --git a/state/project.go b/state/project.go index 89bf52ff..b2186b54 100644 --- a/state/project.go +++ b/state/project.go @@ -10,6 +10,8 @@ import ( "path/filepath" "strings" + "go.dagger.io/dagger/mod" + "github.com/rs/zerolog/log" "go.dagger.io/dagger/keychain" "go.dagger.io/dagger/stdlib" @@ -49,6 +51,7 @@ func Init(ctx context.Context, dir string) (*Project, error) { } return nil, err } + if err := os.Mkdir(path.Join(daggerRoot, envDir), 0755); err != nil { return nil, err } @@ -342,35 +345,35 @@ func (w *Project) cleanPackageName(ctx context.Context, pkg string) (string, err return p, nil } -func cueModInit(ctx context.Context, p string) error { +func cueModInit(ctx context.Context, parentDir string) error { lg := log.Ctx(ctx) - mod := path.Join(p, "cue.mod") - if err := os.Mkdir(mod, 0755); err != nil { + modDir := path.Join(parentDir, "cue.mod") + if err := os.Mkdir(modDir, 0755); err != nil { if !errors.Is(err, os.ErrExist) { return err } } - modFile := path.Join(mod, "module.cue") + modFile := path.Join(modDir, "module.cue") if _, err := os.Stat(modFile); err != nil { if !errors.Is(err, os.ErrNotExist) { return err } - lg.Debug().Str("mod", p).Msg("initializing cue.mod") + lg.Debug().Str("mod", parentDir).Msg("initializing cue.mod") if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil { return err } } - if err := os.Mkdir(path.Join(mod, "usr"), 0755); err != nil { + if err := os.Mkdir(path.Join(modDir, "usr"), 0755); err != nil { if !errors.Is(err, os.ErrExist) { return err } } - if err := os.Mkdir(path.Join(mod, "pkg"), 0755); err != nil { + if err := os.Mkdir(path.Join(modDir, "pkg"), 0755); err != nil { if !errors.Is(err, os.ErrExist) { return err } @@ -379,23 +382,23 @@ func cueModInit(ctx context.Context, p string) error { return nil } -func vendorUniverse(ctx context.Context, p string) error { +func vendorUniverse(ctx context.Context, workspace string) error { // ensure cue module is initialized - if err := cueModInit(ctx, p); err != nil { + if err := cueModInit(ctx, workspace); err != nil { return err } // add universe to `.gitignore` if err := os.WriteFile( - path.Join(p, "cue.mod", "pkg", ".gitignore"), + path.Join(workspace, "cue.mod", "pkg", ".gitignore"), []byte(fmt.Sprintf("# dagger universe\n%s\n", stdlib.PackageName)), 0600, ); err != nil { return err } - log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe") - if err := stdlib.Vendor(ctx, p); err != nil { + log.Ctx(ctx).Debug().Str("mod", workspace).Msg("vendoring universe") + if err := mod.InstallStdlib(workspace); err != nil { return err } diff --git a/state/state.go b/state/state.go index 918d19bb..63e1767d 100644 --- a/state/state.go +++ b/state/state.go @@ -35,8 +35,8 @@ type State struct { func (s *State) CompilePlan(ctx context.Context) (*compiler.Value, error) { w := s.Project // FIXME: backward compatibility - if mod := s.Plan.Module; mod != "" { - w = path.Join(w, mod) + if planModule := s.Plan.Module; planModule != "" { + w = path.Join(w, planModule) } // FIXME: universe vendoring @@ -51,7 +51,7 @@ func (s *State) CompilePlan(ctx context.Context) (*compiler.Value, error) { return nil, err } - args := []string{} + var args []string if pkg := s.Plan.Package; pkg != "" { args = append(args, pkg) } @@ -81,21 +81,6 @@ func (s *State) CompileInputs() (*compiler.Value, error) { return v, nil } -// VendorUniverse vendors the latest (built-in) version of the universe into the -// environment's `cue.mod`. -// FIXME: This has nothing to do in `State` and should be tied to a `Project`. -// However, since environments could point to different modules before, we have -// to handle vendoring on a per environment basis. -func (s *State) VendorUniverse(ctx context.Context) error { - w := s.Project - // FIXME: backward compatibility - if mod := s.Plan.Module; mod != "" { - w = path.Join(w, mod) - } - - return vendorUniverse(ctx, w) -} - type Plan struct { Module string `yaml:"module,omitempty"` Package string `yaml:"package,omitempty"` diff --git a/state/state_test.go b/state/state_test.go new file mode 100644 index 00000000..7bf2df5b --- /dev/null +++ b/state/state_test.go @@ -0,0 +1 @@ +package state