kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Merge pull request #934 from tjovicic/vendoring-improved

Browse Source 
Use package manager when vendoring stdlib
This commit is contained in:
Sam Alba 2021-10-13 20:30:17 -07:00 committed by GitHub
commit 63bb368d08
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 704 additions and 423 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -27,3 +27,7 @@
tests/node_modules tests/node_modules
dist/ dist/
docs/learn/tests/node_modules docs/learn/tests/node_modules
# Integration CI dagger.mod, dagger.sum
dagger.mod
dagger.sum

57
cmd/dagger/cmd/mod/arg.go
View File

@ -1,57 +0,0 @@
package mod
import (
"fmt"
"path"
"regexp"
"strings"
)
func parseArgument(arg string) (*require, error) {
switch {
case strings.HasPrefix(arg, "github.com"):
return parseGithubRepoName(arg)
case strings.HasPrefix(arg, "alpha.dagger.io"):
return parseDaggerRepoName(arg)
default:
return nil, fmt.Errorf("repo name does not match suported providers")
}
}
var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseGithubRepoName(arg string) (*require, error) {
repoMatches := githubRepoNameRegex.FindStringSubmatch(arg)
if len(repoMatches) < 4 {
return nil, fmt.Errorf("issue when parsing github repo")
}
return &require{
repo: repoMatches[1],
path: repoMatches[2],
version: repoMatches[3],
cloneRepo: repoMatches[1],
clonePath: repoMatches[2],
}, nil
}
var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseDaggerRepoName(arg string) (*require, error) {
repoMatches := daggerRepoNameRegex.FindStringSubmatch(arg)
if len(repoMatches) < 3 {
return nil, fmt.Errorf("issue when parsing dagger repo")
}
return &require{
repo: "alpha.dagger.io",
path: repoMatches[1],
version: repoMatches[2],
cloneRepo: "github.com/dagger/dagger",
clonePath: path.Join("/stdlib", repoMatches[1]),
}, nil
}

223
cmd/dagger/cmd/mod/file.go
View File

@ -1,223 +0,0 @@
package mod
import (
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"io/ioutil"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"github.com/spf13/viper"
)
const filePath = "./cue.mod/dagger.mod"
const destBasePath = "./cue.mod/pkg"
const tmpBasePath = "./cue.mod/tmp"
// A file is the parsed, interpreted form of a cue.mod file.
type file struct {
require []*require
projectPath string
}
func readPath(projectPath string) (*file, error) {
p := path.Join(projectPath, filePath)
f, err := os.Open(p)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return nil, err
}
// dagger.mod.cue doesn't exist, let's create an empty file
if f, err = os.Create(p); err != nil {
return nil, err
}
}
modFile, err := read(f)
if err != nil {
return nil, err
}
modFile.projectPath = projectPath
return modFile, nil
}
func read(f io.Reader) (*file, error) {
b, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
lines := nonEmptyLines(b)
requires := make([]*require, 0, len(lines))
for _, line := range lines {
split := strings.Split(line, " ")
r, err := parseArgument(split[0])
if err != nil {
return nil, err
}
r.version = split[1]
requires = append(requires, r)
}
return &file{
require: requires,
}, nil
}
var spaceRgx = regexp.MustCompile(`\s+`)
func nonEmptyLines(b []byte) []string {
s := strings.ReplaceAll(string(b), "\r\n", "\n")
split := strings.Split(s, "\n")
lines := make([]string, 0, len(split))
for _, l := range split {
trimmed := strings.TrimSpace(l)
if trimmed == "" {
continue
}
trimmed = spaceRgx.ReplaceAllString(trimmed, " ")
lines = append(lines, trimmed)
}
return lines
}
func (f *file) processRequire(req *require, upgrade bool) (bool, error) {
var isNew bool
tmpPath := path.Join(f.projectPath, tmpBasePath, req.repo)
if err := os.MkdirAll(tmpPath, 0755); err != nil {
return false, fmt.Errorf("error creating tmp dir for cloning package")
}
defer os.RemoveAll(tmpPath)
// clone the repo
privateKeyFile := viper.GetString("private-key-file")
privateKeyPassword := viper.GetString("private-key-password")
r, err := clone(req, tmpPath, privateKeyFile, privateKeyPassword)
if err != nil {
return isNew, fmt.Errorf("error downloading package %s: %w", req, err)
}
existing := f.search(req)
destPath := path.Join(f.projectPath, destBasePath)
// requirement is new, so we should move the files and add it to the mod file
if existing == nil {
if err := move(req, tmpPath, destPath); err != nil {
return isNew, err
}
f.require = append(f.require, req)
isNew = true
return isNew, nil
}
if upgrade {
latestTag, err := r.latestTag()
if err != nil {
return isNew, err
}
if latestTag == "" {
return isNew, fmt.Errorf("repo does not have a tag")
}
req.version = latestTag
}
c, err := compareVersions(existing.version, req.version)
if err != nil {
return isNew, err
}
// the existing requirement is newer or equal so we skip installation
if c >= 0 {
return isNew, nil
}
// the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing
// requirement and replace the code in the /pkg folder
existing.version = req.version
if err = r.checkout(req.version); err != nil {
return isNew, err
}
if err = replace(req, tmpPath, destPath); err != nil {
return isNew, err
}
isNew = true
return isNew, nil
}
func (f *file) write() error {
return ioutil.WriteFile(path.Join(f.projectPath, filePath), f.contents().Bytes(), 0600)
}
func (f *file) contents() *bytes.Buffer {
var b bytes.Buffer
for _, r := range f.require {
b.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.version))
}
return &b
}
func (f *file) search(r *require) *require {
for _, existing := range f.require {
if existing.fullPath() == r.fullPath() {
return existing
}
}
return nil
}
type require struct {
repo string
path string
version string
cloneRepo string
clonePath string
}
func (r *require) fullPath() string {
return path.Join(r.repo, r.path)
}
func move(r *require, sourceRepoPath, destBasePath string) error {
destPath := path.Join(destBasePath, r.fullPath())
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
return err
}
if err := os.Rename(path.Join(sourceRepoPath, r.path), destPath); err != nil {
return err
}
return nil
}
func replace(r *require, sourceRepoPath, destBasePath string) error {
if err := os.RemoveAll(path.Join(destBasePath, r.fullPath())); err != nil {
return err
}
return move(r, sourceRepoPath, destBasePath)
}

80
cmd/dagger/cmd/mod/get.go
View File

@ -1,11 +1,11 @@
package mod package mod
import ( import (
"github.com/hashicorp/go-version"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"go.dagger.io/dagger/cmd/dagger/cmd/common" "go.dagger.io/dagger/cmd/dagger/cmd/common"
"go.dagger.io/dagger/cmd/dagger/logger" "go.dagger.io/dagger/cmd/dagger/logger"
"go.dagger.io/dagger/mod"
"go.dagger.io/dagger/telemetry" "go.dagger.io/dagger/telemetry"
) )
@ -32,77 +32,41 @@ var getCmd = &cobra.Command{
Value: args, Value: args,
}) })
// read mod file in the current dir var update = viper.GetBool("update")
modFile, err := readPath(project.Path)
if err != nil {
lg.Fatal().Err(err).Msg("error loading module file")
}
// parse packages to install var processedRequires []*mod.Require
var packages []*require var err error
var upgrade bool if update && len(args) == 0 {
lg.Info().Msg("updating all installed packages...")
if len(args) == 0 { processedRequires, err = mod.UpdateInstalled(project.Path)
lg.Info().Msg("upgrading installed packages...") } else if update && len(args) > 0 {
packages = modFile.require lg.Info().Msg("updating specified packages...")
upgrade = true processedRequires, err = mod.UpdateAll(project.Path, args)
} else if !update && len(args) > 0 {
lg.Info().Msg("installing specified packages...")
processedRequires, err = mod.InstallAll(project.Path, args)
} else { } else {
for _, arg := range args { lg.Fatal().Msg("unrecognized update/install operation")
p, err := parseArgument(arg) }
if err != nil {
lg.Error().Err(err).Msgf("error parsing package %s", arg) if len(processedRequires) > 0 {
continue for _, r := range processedRequires {
} lg.Info().Msgf("installed/updated package %s", r)
packages = append(packages, p)
} }
} }
// download packages if err != nil {
for _, p := range packages { lg.Error().Err(err).Msg("error installing/updating packages")
isNew, err := modFile.processRequire(p, upgrade)
if err != nil {
lg.Error().Err(err).Msgf("error processing package %s", p.repo)
}
if isNew {
lg.Info().Msgf("downloading %s:%v", p.repo, p.version)
}
}
// write to mod file in the current dir
if err = modFile.write(); err != nil {
lg.Error().Err(err).Msg("error writing to mod file")
} }
<-doneCh <-doneCh
}, },
} }
func compareVersions(reqV1, reqV2 string) (int, error) {
v1, err := version.NewVersion(reqV1)
if err != nil {
return 0, err
}
v2, err := version.NewVersion(reqV2)
if err != nil {
return 0, err
}
if v1.LessThan(v2) {
return -1, nil
}
if v1.Equal(v2) {
return 0, nil
}
return 1, nil
}
func init() { func init() {
getCmd.Flags().String("private-key-file", "", "Private ssh key") getCmd.Flags().String("private-key-file", "", "Private ssh key")
getCmd.Flags().String("private-key-password", "", "Private ssh key password") getCmd.Flags().String("private-key-password", "", "Private ssh key password")
getCmd.Flags().BoolP("update", "u", false, "Update specified package")
if err := viper.BindPFlags(getCmd.Flags()); err != nil { if err := viper.BindPFlags(getCmd.Flags()); err != nil {
panic(err) panic(err)

8
docs/learn/1010-dev-cue-package.md
View File

@ -42,13 +42,13 @@ domain name (as in Go) followed by a descriptive name. In this example, we reuse
package from it. package from it.
```shell ```shell
mkdir -p cue.mod/pkg/github.com/tjovicic/gcpcloudrun mkdir -p cue.mod/pkg/github.com/username/gcpcloudrun
``` ```
Let's write the package logic. It is basically what we've seen in the 106-cloudrun example: Let's write the package logic. It is basically what we've seen in the 106-cloudrun example:
```shell ```shell
touch cue.mod/pkg/github.com/tjovicic/gcpcloudrun/source.cue touch cue.mod/pkg/github.com/username/gcpcloudrun/source.cue
``` ```
```cue file=./tests/dev-cue-package/source.cue title="cue.mod/pkg/github.com/tjovicic/gcpcloudrun/source.cue" ```cue file=./tests/dev-cue-package/source.cue title="cue.mod/pkg/github.com/tjovicic/gcpcloudrun/source.cue"
@ -86,8 +86,8 @@ You've probably guessed this package isn't tied to just your project. You can ea
of different projects and use it as we've shown above. of different projects and use it as we've shown above.
```shell ```shell
mkdir -p /my-new-project/cue.mod/pkg/github.com/tjovicic/gcpcloudrun mkdir -p /my-new-workspace/cue.mod/pkg/github.com/username/gcpcloudrun
cp ./cue.mod/pkg/github.com/tjovicic/gcpcloudrun/source.cue /new-project/cue.mod/pkg/github.com/tjovicic/gcpcloudrun cp ./cue.mod/pkg/github.com/username/gcpcloudrun/source.cue /new-workspace/cue.mod/pkg/github.com/username/gcpcloudrun
``` ```
## Contributing to Dagger stdlib ## Contributing to Dagger stdlib

4
docs/learn/1011-package-manager.md
View File

@ -59,7 +59,7 @@ dagger mod get github.com/dagger/packages/gcpcloudrun@v0.1
``` ```
It should pull the `v0.1` version from Github, leave a copy in `cue.mod/pkg` and reflect the change in It should pull the `v0.1` version from Github, leave a copy in `cue.mod/pkg` and reflect the change in
`cue.mod/dagger.mod.cue` file: `cue.mod/dagger.mod` file:
```shell ```shell
cue.mod/pkg/github.com/ cue.mod/pkg/github.com/
@ -111,7 +111,7 @@ You should see similar output:
12:25PM INF system | downloading github.com/dagger/packages:v0.2 12:25PM INF system | downloading github.com/dagger/packages:v0.2
``` ```
And `cue.mod/dagger.mod.cue` should reflect the new version: And `cue.mod/dagger.mod` should reflect the new version:
```cue title="./cue.mod/dagger.mod" ```cue title="./cue.mod/dagger.mod"
github.com/dagger/packages/gcpcloudrun v0.2 github.com/dagger/packages/gcpcloudrun v0.2

2
go.mod
View File

@ -11,6 +11,7 @@ require (
github.com/docker/distribution v2.7.1+incompatible github.com/docker/distribution v2.7.1+incompatible
github.com/emicklei/proto v1.9.0 // indirect github.com/emicklei/proto v1.9.0 // indirect
github.com/go-git/go-git/v5 v5.4.2 github.com/go-git/go-git/v5 v5.4.2
github.com/gofrs/flock v0.8.1
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/google/uuid v1.3.0 github.com/google/uuid v1.3.0
github.com/hashicorp/go-version v1.3.0 github.com/hashicorp/go-version v1.3.0
@ -34,6 +35,7 @@ require (
go.opentelemetry.io/otel/sdk v1.0.1 go.opentelemetry.io/otel/sdk v1.0.1
go.opentelemetry.io/otel/trace v1.0.1 go.opentelemetry.io/otel/trace v1.0.1
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
golang.org/x/mod v0.4.2
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef // indirect golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b

4
go.sum
View File

@ -594,8 +594,9 @@ github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gofrs/flock v0.7.3 h1:I0EKY9l8HZCXTMYC4F80vwT6KNypV9uYKP3Alm/hjmQ=
github.com/gofrs/flock v0.7.3/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/flock v0.7.3/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
github.com/gogo/googleapis v1.3.2/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/googleapis v1.3.2/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
@ -1503,6 +1504,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

31
mod/checksum.go Normal file
View File

@ -0,0 +1,31 @@
package mod
import (
"fmt"
"os"
"path"
"golang.org/x/mod/sumdb/dirhash"
)
func dirChecksum(dirPath string) (string, error) {
err := cleanDirForChecksum(dirPath)
if err != nil {
return "", err
}
checksum, err := dirhash.HashDir(dirPath, "", dirhash.DefaultHash)
if err != nil {
return "", err
}
return checksum, nil
}
func cleanDirForChecksum(dirPath string) error {
if err := os.RemoveAll(path.Join(dirPath, ".git")); err != nil {
return fmt.Errorf("error cleaning up .git directory")
}
return nil
}

293
mod/file.go Normal file
View File

@ -0,0 +1,293 @@
package mod
import (
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"io/ioutil"
"os"
"path"
"regexp"
"strings"
"github.com/spf13/viper"
)
const modFilePath = "./cue.mod/dagger.mod"
const sumFilePath = "./cue.mod/dagger.sum"
const lockFilePath = "./cue.mod/dagger.lock"
const destBasePath = "./cue.mod/pkg"
const tmpBasePath = "./cue.mod/tmp"
// file is the parsed, interpreted form of dagger.mod file.
type file struct {
requires []*Require
workspacePath string
}
func readPath(workspacePath string) (*file, error) {
pMod := path.Join(workspacePath, modFilePath)
fMod, err := os.Open(pMod)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return nil, err
}
// dagger.mod doesn't exist, let's create an empty file
if fMod, err = os.Create(pMod); err != nil {
return nil, err
}
}
pSum := path.Join(workspacePath, sumFilePath)
fSum, err := os.Open(pSum)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return nil, err
}
// dagger.sum doesn't exist, let's create an empty file
if fSum, err = os.Create(pSum); err != nil {
return nil, err
}
}
modFile, err := read(fMod, fSum)
if err != nil {
return nil, err
}
modFile.workspacePath = workspacePath
return modFile, nil
}
func read(fMod, fSum io.Reader) (*file, error) {
bMod, err := ioutil.ReadAll(fMod)
if err != nil {
return nil, err
}
bSum, err := ioutil.ReadAll(fSum)
if err != nil {
return nil, err
}
modLines := nonEmptyLines(bMod)
sumLines := nonEmptyLines(bSum)
if len(modLines) != len(sumLines) {
return nil, fmt.Errorf("length of dagger.mod and dagger.sum files differ")
}
requires := make([]*Require, 0, len(modLines))
for i := 0; i < len(modLines); i++ {
modSplit := strings.Split(modLines[i], " ")
if len(modSplit) != 2 {
return nil, fmt.Errorf("line in the mod file doesn't contain 2 elements")
}
sumSplit := strings.Split(sumLines[i], " ")
if len(sumSplit) != 2 {
return nil, fmt.Errorf("line in the sum file doesn't contain 2 elements")
}
if modSplit[0] != sumSplit[0] {
return nil, fmt.Errorf("repos in mod and sum line don't match: %s and %s", modSplit[0], sumSplit[0])
}
require, err := newRequire(modSplit[0])
if err != nil {
return nil, err
}
require.version = modSplit[1]
require.checksum = sumSplit[1]
requires = append(requires, require)
}
return &file{requires: requires}, nil
}
var spaceRgx = regexp.MustCompile(`\s+`)
func nonEmptyLines(b []byte) []string {
s := strings.ReplaceAll(string(b), "\r\n", "\n")
split := strings.Split(s, "\n")
lines := make([]string, 0, len(split))
for _, l := range split {
trimmed := strings.TrimSpace(l)
if trimmed == "" {
continue
}
trimmed = spaceRgx.ReplaceAllString(trimmed, " ")
lines = append(lines, trimmed)
}
return lines
}
func (f *file) install(req *Require) error {
// cleaning up possible leftovers
tmpPath := path.Join(f.workspacePath, tmpBasePath, req.fullPath())
defer os.RemoveAll(tmpPath)
// clone to a tmp directory
r, err := clone(req, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password"))
if err != nil {
return fmt.Errorf("error downloading package %s: %w", req, err)
}
destPath := path.Join(f.workspacePath, destBasePath, req.fullPath())
// requirement is new, so we should move the cloned files from tmp to pkg and add it to the mod file
existing := f.searchInstalledRequire(req)
if existing == nil {
if err = replace(req, tmpPath, destPath); err != nil {
return err
}
checksum, err := dirChecksum(destPath)
if err != nil {
return err
}
req.checksum = checksum
f.requires = append(f.requires, req)
return nil
}
// checkout the cloned repo to that tag, change the version in the existing requirement and
// replace the code in the /pkg folder
existing.version = req.version
if err = r.checkout(req.version); err != nil {
return err
}
if err = replace(req, tmpPath, destPath); err != nil {
return err
}
checksum, err := dirChecksum(destPath)
if err != nil {
return err
}
existing.checksum = checksum
return nil
}
func (f *file) updateToLatest(req *Require) (*Require, error) {
// check if it doesn't exist
existing := f.searchInstalledRequire(req)
if existing == nil {
return nil, fmt.Errorf("package %s isn't already installed", req.fullPath())
}
// cleaning up possible leftovers
tmpPath := path.Join(f.workspacePath, tmpBasePath, existing.fullPath())
defer os.RemoveAll(tmpPath)
// clone to a tmp directory
gitRepo, err := clone(existing, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password"))
if err != nil {
return nil, fmt.Errorf("error downloading package %s: %w", existing, err)
}
// checkout the latest tag
latestTag, err := gitRepo.latestTag()
if err != nil {
return nil, err
}
c, err := compareVersions(latestTag, existing.version)
if err != nil {
return nil, err
}
if c < 0 {
return nil, fmt.Errorf("latest git tag is less than the current version")
}
existing.version = latestTag
if err = gitRepo.checkout(existing.version); err != nil {
return nil, err
}
// move the package from tmp to pkg directory
destPath := path.Join(f.workspacePath, destBasePath, existing.fullPath())
if err = replace(existing, tmpPath, destPath); err != nil {
return nil, err
}
checksum, err := dirChecksum(destPath)
if err != nil {
return nil, err
}
req.checksum = checksum
return existing, nil
}
func (f *file) searchInstalledRequire(r *Require) *Require {
for _, existing := range f.requires {
if existing.fullPath() == r.fullPath() {
return existing
}
}
return nil
}
func (f *file) ensure() error {
for _, require := range f.requires {
requirePath := path.Join(f.workspacePath, destBasePath, require.fullPath())
checksum, err := dirChecksum(requirePath)
if err != nil {
return nil
}
if require.checksum != checksum {
return fmt.Errorf("wrong checksum for %s", require.fullPath())
}
}
return nil
}
func (f *file) write() error {
// write dagger.mod file
var bMod bytes.Buffer
for _, r := range f.requires {
bMod.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.version))
}
err := ioutil.WriteFile(path.Join(f.workspacePath, modFilePath), bMod.Bytes(), 0600)
if err != nil {
return err
}
// write dagger.sum file
var bSum bytes.Buffer
for _, r := range f.requires {
bSum.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.checksum))
}
err = ioutil.WriteFile(path.Join(f.workspacePath, sumFilePath), bSum.Bytes(), 0600)
if err != nil {
return err
}
return nil
}

21
cmd/dagger/cmd/mod/file_test.go → mod/file_test.go
View File

@ -7,18 +7,23 @@ import (
func TestReadFile(t *testing.T) { func TestReadFile(t *testing.T) {
cases := []struct { cases := []struct {
name string name string
input string modFile string
want *file sumFile string
want *file
}{ }{
{ {
name: "module file with valid dependencies", name: "module file with valid dependencies",
input: ` modFile: `
github.com/tjovicic/test xyz github.com/tjovicic/test xyz
github.com/bla/bla abc github.com/bla/bla abc
`, `,
sumFile: `
github.com/tjovicic/test h1:hash
github.com/bla/bla h1:hash
`,
want: &file{ want: &file{
require: []*require{ requires: []*Require{
{ {
repo: "github.com/tjovicic/test", repo: "github.com/tjovicic/test",
path: "", path: "",
@ -36,13 +41,13 @@ func TestReadFile(t *testing.T) {
for _, c := range cases { for _, c := range cases {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
got, err := read(strings.NewReader(c.input)) got, err := read(strings.NewReader(c.modFile), strings.NewReader(c.sumFile))
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
if len(got.require) != len(c.want.require) { if len(got.requires) != len(c.want.requires) {
t.Errorf("requires length differs: want %d, got %d", len(c.want.require), len(got.require)) t.Errorf("requires length differs: want %d, got %d", len(c.want.requires), len(got.requires))
} }
}) })
} }

135
mod/mod.go Normal file
View File

@ -0,0 +1,135 @@
package mod
import (
"path"
"github.com/gofrs/flock"
)
func InstallStdlib(workspace string) error {
_, err := Install(workspace, "alpha.dagger.io@v0.1")
return err
}
func Install(workspace, repoName string) (*Require, error) {
require, err := newRequire(repoName)
if err != nil {
return nil, err
}
modfile, err := readPath(workspace)
if err != nil {
return nil, err
}
fileLock := flock.New(path.Join(workspace, lockFilePath))
if err := fileLock.Lock(); err != nil {
return nil, err
}
err = modfile.install(require)
if err != nil {
return nil, err
}
if err = modfile.write(); err != nil {
return nil, err
}
if err := fileLock.Unlock(); err != nil {
return nil, err
}
return require, nil
}
func InstallAll(workspace string, repoNames []string) ([]*Require, error) {
installedRequires := make([]*Require, 0, len(repoNames))
var err error
for _, repoName := range repoNames {
var require *Require
if require, err = Install(workspace, repoName); err != nil {
continue
}
installedRequires = append(installedRequires, require)
}
return installedRequires, err
}
func Update(workspace, repoName string) (*Require, error) {
require, err := newRequire(repoName)
if err != nil {
return nil, err
}
modfile, err := readPath(workspace)
if err != nil {
return nil, err
}
fileLock := flock.New(path.Join(workspace, lockFilePath))
if err := fileLock.Lock(); err != nil {
return nil, err
}
updatedRequire, err := modfile.updateToLatest(require)
if err != nil {
return nil, err
}
if err = modfile.write(); err != nil {
return nil, err
}
if err := fileLock.Unlock(); err != nil {
return nil, err
}
return updatedRequire, nil
}
func UpdateAll(workspace string, repoNames []string) ([]*Require, error) {
updatedRequires := make([]*Require, 0, len(repoNames))
var err error
for _, repoName := range repoNames {
var require *Require
if require, err = Update(workspace, repoName); err != nil {
continue
}
updatedRequires = append(updatedRequires, require)
}
return updatedRequires, err
}
func UpdateInstalled(workspace string) ([]*Require, error) {
modfile, err := readPath(workspace)
if err != nil {
return nil, err
}
repoNames := make([]string, 0, len(modfile.requires))
for _, require := range modfile.requires {
repoNames = append(repoNames, require.String())
}
return UpdateAll(workspace, repoNames)
}
func Ensure(workspace string) error {
modfile, err := readPath(workspace)
if err != nil {
return err
}
return modfile.ensure()
}

30
cmd/dagger/cmd/mod/repo.go → mod/repo.go
View File

@ -3,7 +3,6 @@ package mod
import ( import (
"fmt" "fmt"
"os" "os"
"path"
"sort" "sort"
"strings" "strings"
@ -15,11 +14,17 @@ import (
) )
type repo struct { type repo struct {
localPath string contents *git.Repository
contents *git.Repository
} }
func clone(require *require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) { func clone(require *Require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) {
if err := os.RemoveAll(dir); err != nil {
return nil, fmt.Errorf("error cleaning up tmp directory")
}
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, fmt.Errorf("error creating tmp dir for cloning package")
}
o := git.CloneOptions{ o := git.CloneOptions{
URL: fmt.Sprintf("https://%s", require.cloneRepo), URL: fmt.Sprintf("https://%s", require.cloneRepo),
} }
@ -40,8 +45,7 @@ func clone(require *require, dir string, privateKeyFile, privateKeyPassword stri
} }
rr := &repo{ rr := &repo{
localPath: dir, contents: r,
contents: r,
} }
if require.version == "" { if require.version == "" {
@ -50,10 +54,6 @@ func clone(require *require, dir string, privateKeyFile, privateKeyPassword stri
return nil, err return nil, err
} }
if latestTag == "" {
return nil, fmt.Errorf("no git tags found in the repo")
}
require.version = latestTag require.version = latestTag
} }
@ -61,10 +61,6 @@ func clone(require *require, dir string, privateKeyFile, privateKeyPassword stri
return nil, err return nil, err
} }
if _, err := os.Stat(path.Join(dir, require.clonePath, filePath)); err != nil {
return nil, fmt.Errorf("repo does not contain %s", filePath)
}
return rr, nil return rr, nil
} }
@ -118,11 +114,11 @@ func (r *repo) latestTag() (string, error) {
versions[i] = v versions[i] = v
} }
sort.Sort(version.Collection(versions))
if len(versions) == 0 { if len(versions) == 0 {
return "", nil return "", fmt.Errorf("repo doesn't have any tags")
} }
sort.Sort(version.Collection(versions))
return versions[len(versions)-1].Original(), nil return versions[len(versions)-1].Original(), nil
} }

34
cmd/dagger/cmd/mod/repo_test.go → mod/repo_test.go
View File

@ -9,37 +9,37 @@ import (
func TestClone(t *testing.T) { func TestClone(t *testing.T) {
cases := []struct { cases := []struct {
name string name string
require require require Require
privateKeyFile string privateKeyFile string
privateKeyPassword string privateKeyPassword string
}{ }{
{ {
name: "resolving shorter hash version", name: "resolving shorter hash version",
require: require{ require: Require{
cloneRepo: "github.com/tjovicic/dagger-modules", cloneRepo: "github.com/dagger/universe",
clonePath: "gcpcloudrun", clonePath: "stdlib",
version: "26a1d46d1b3c", version: "24d7af3fc2a3e9c7cc2",
}, },
}, },
{ {
name: "resolving branch name", name: "resolving branch name",
require: require{ require: Require{
cloneRepo: "github.com/tjovicic/dagger-modules", cloneRepo: "github.com/dagger/universe",
clonePath: "gcpcloudrun", clonePath: "stdlib",
version: "main", version: "main",
}, },
}, },
{ {
name: "resolving tag", name: "resolving tag",
require: require{ require: Require{
cloneRepo: "github.com/tjovicic/dagger-modules", cloneRepo: "github.com/dagger/universe",
clonePath: "gcpcloudrun", clonePath: "stdlib",
version: "v0.3", version: "v0.1",
}, },
}, },
{ {
name: "Dagger private test repo", name: "dagger private repo",
require: require{ require: Require{
cloneRepo: "github.com/dagger/test", cloneRepo: "github.com/dagger/test",
clonePath: "", clonePath: "",
version: "main", version: "main",
@ -73,9 +73,9 @@ func TestListTags(t *testing.T) {
} }
defer os.Remove(tmpDir) defer os.Remove(tmpDir)
r, err := clone(&require{ r, err := clone(&Require{
cloneRepo: "github.com/tjovicic/dagger-modules", cloneRepo: "github.com/dagger/universe",
clonePath: "gcpcloudrun", clonePath: "stdlib",
version: "", version: "",
}, tmpDir, "", "") }, tmpDir, "", "")
if err != nil { if err != nil {

90
mod/require.go Normal file
View File

@ -0,0 +1,90 @@
package mod
import (
"fmt"
"os"
"path"
"regexp"
"strings"
)
type Require struct {
repo string
path string
cloneRepo string
clonePath string
version string
checksum string
}
func newRequire(repoName string) (*Require, error) {
switch {
case strings.HasPrefix(repoName, "github.com"):
return parseGithubRepoName(repoName)
case strings.HasPrefix(repoName, "alpha.dagger.io"):
return parseDaggerRepoName(repoName)
default:
return nil, fmt.Errorf("repo name does not match suported providers")
}
}
var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseGithubRepoName(repoName string) (*Require, error) {
repoMatches := githubRepoNameRegex.FindStringSubmatch(repoName)
if len(repoMatches) < 4 {
return nil, fmt.Errorf("issue when parsing github repo")
}
return &Require{
repo: repoMatches[1],
path: repoMatches[2],
version: repoMatches[3],
cloneRepo: repoMatches[1],
clonePath: repoMatches[2],
}, nil
}
var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseDaggerRepoName(repoName string) (*Require, error) {
repoMatches := daggerRepoNameRegex.FindStringSubmatch(repoName)
if len(repoMatches) < 3 {
return nil, fmt.Errorf("issue when parsing dagger repo")
}
return &Require{
repo: "alpha.dagger.io",
path: repoMatches[1],
version: repoMatches[2],
cloneRepo: "github.com/dagger/universe",
clonePath: path.Join("/stdlib", repoMatches[1]),
}, nil
}
func (r *Require) String() string {
return fmt.Sprintf("%s@%s", r.fullPath(), r.version)
}
func (r *Require) fullPath() string {
return path.Join(r.repo, r.path)
}
func replace(r *Require, sourceRepoPath, destPath string) error {
// remove previous package directory
if err := os.RemoveAll(destPath); err != nil {
return err
}
if err := os.Rename(path.Join(sourceRepoPath, r.clonePath), destPath); err != nil {
return err
}
return nil
}

45
cmd/dagger/cmd/mod/arg_test.go → mod/require_test.go
View File

@ -1,12 +1,14 @@
package mod package mod
import "testing" import (
"testing"
)
func TestParseArgument(t *testing.T) { func TestParseArgument(t *testing.T) {
cases := []struct { cases := []struct {
name string name string
in string in string
want *require want *Require
hasError bool hasError bool
}{ }{
{ {
@ -17,7 +19,7 @@ func TestParseArgument(t *testing.T) {
{ {
name: "Dagger repo", name: "Dagger repo",
in: "github.com/dagger/dagger", in: "github.com/dagger/dagger",
want: &require{ want: &Require{
repo: "github.com/dagger/dagger", repo: "github.com/dagger/dagger",
path: "", path: "",
version: "", version: "",
@ -26,7 +28,7 @@ func TestParseArgument(t *testing.T) {
{ {
name: "Dagger repo with path", name: "Dagger repo with path",
in: "github.com/dagger/dagger/stdlib", in: "github.com/dagger/dagger/stdlib",
want: &require{ want: &Require{
repo: "github.com/dagger/dagger", repo: "github.com/dagger/dagger",
path: "/stdlib", path: "/stdlib",
version: "", version: "",
@ -35,7 +37,7 @@ func TestParseArgument(t *testing.T) {
{ {
name: "Dagger repo with longer path", name: "Dagger repo with longer path",
in: "github.com/dagger/dagger/stdlib/test/test", in: "github.com/dagger/dagger/stdlib/test/test",
want: &require{ want: &Require{
repo: "github.com/dagger/dagger", repo: "github.com/dagger/dagger",
path: "/stdlib/test/test", path: "/stdlib/test/test",
version: "", version: "",
@ -44,25 +46,25 @@ func TestParseArgument(t *testing.T) {
{ {
name: "Dagger repo with path and version", name: "Dagger repo with path and version",
in: "github.com/dagger/dagger/stdlib@v0.1", in: "github.com/dagger/dagger/stdlib@v0.1",
want: &require{ want: &Require{
repo: "github.com/dagger/dagger", repo: "github.com/dagger/dagger",
path: "/stdlib", path: "/stdlib",
version: "v0.1", version: "v0.1",
}, },
}, },
{ {
name: "Dagger repo with longer path and version", name: "Dagger repo with longer path and version tag",
in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1", in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1",
want: &require{ want: &Require{
repo: "github.com/dagger/dagger", repo: "github.com/dagger/dagger",
path: "/stdlib/test/test", path: "/stdlib/test/test",
version: "v0.0.1", version: "v0.0.1",
}, },
}, },
{ {
name: "Alpha Dagger repo", name: "Alpha Dagger repo with path",
in: "alpha.dagger.io/gcp/gke@v0.1.0-alpha.20", in: "alpha.dagger.io/gcp/gke@v0.1.0-alpha.20",
want: &require{ want: &Require{
repo: "alpha.dagger.io", repo: "alpha.dagger.io",
path: "/gcp/gke", path: "/gcp/gke",
version: "v0.1.0-alpha.20", version: "v0.1.0-alpha.20",
@ -71,11 +73,32 @@ func TestParseArgument(t *testing.T) {
clonePath: "/stdlib/gcp/gke", clonePath: "/stdlib/gcp/gke",
}, },
}, },
{
name: "Alpha Dagger repo",
in: "alpha.dagger.io@v0.1.0-alpha.23",
want: &Require{
repo: "alpha.dagger.io",
path: "",
version: "v0.1.0-alpha.23",
cloneRepo: "github.com/dagger/dagger",
clonePath: "/stdlib",
},
},
{
name: "Dagger repo with longer path and commit version",
in: "github.com/dagger/dagger/stdlib/test/test@26a1d46d1b3c",
want: &Require{
repo: "github.com/dagger/dagger",
path: "/stdlib/test/test",
version: "26a1d46d1b3c",
},
},
} }
for _, c := range cases { for _, c := range cases {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
got, err := parseArgument(c.in) got, err := newRequire(c.in)
if err != nil && c.hasError { if err != nil && c.hasError {
return return
} }

0
cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test → mod/test-ssh-keys/id_ed25519_test
View File

0
cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test.pub → mod/test-ssh-keys/id_ed25519_test.pub
View File

27
mod/version.go Normal file
View File

@ -0,0 +1,27 @@
package mod
import "github.com/hashicorp/go-version"
// compareVersions returns -1 if the first argument is less or 1 if it's greater than the second argument.
// It returns 0 if they are equal.
func compareVersions(reqV1, reqV2 string) (int, error) {
v1, err := version.NewVersion(reqV1)
if err != nil {
return 0, err
}
v2, err := version.NewVersion(reqV2)
if err != nil {
return 0, err
}
if v1.LessThan(v2) {
return -1, nil
}
if v1.Equal(v2) {
return 0, nil
}
return 1, nil
}

17
state/project.go
View File

@ -49,6 +49,7 @@ func Init(ctx context.Context, dir string) (*Project, error) {
} }
return nil, err return nil, err
} }
if err := os.Mkdir(path.Join(daggerRoot, envDir), 0755); err != nil { if err := os.Mkdir(path.Join(daggerRoot, envDir), 0755); err != nil {
return nil, err return nil, err
} }
@ -342,35 +343,35 @@ func (w *Project) cleanPackageName(ctx context.Context, pkg string) (string, err
return p, nil return p, nil
} }
func cueModInit(ctx context.Context, p string) error { func cueModInit(ctx context.Context, parentDir string) error {
lg := log.Ctx(ctx) lg := log.Ctx(ctx)
mod := path.Join(p, "cue.mod") modDir := path.Join(parentDir, "cue.mod")
if err := os.Mkdir(mod, 0755); err != nil { if err := os.Mkdir(modDir, 0755); err != nil {
if !errors.Is(err, os.ErrExist) { if !errors.Is(err, os.ErrExist) {
return err return err
} }
} }
modFile := path.Join(mod, "module.cue") modFile := path.Join(modDir, "module.cue")
if _, err := os.Stat(modFile); err != nil { if _, err := os.Stat(modFile); err != nil {
if !errors.Is(err, os.ErrNotExist) { if !errors.Is(err, os.ErrNotExist) {
return err return err
} }
lg.Debug().Str("mod", p).Msg("initializing cue.mod") lg.Debug().Str("mod", parentDir).Msg("initializing cue.mod")
if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil { if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil {
return err return err
} }
} }
if err := os.Mkdir(path.Join(mod, "usr"), 0755); err != nil { if err := os.Mkdir(path.Join(modDir, "usr"), 0755); err != nil {
if !errors.Is(err, os.ErrExist) { if !errors.Is(err, os.ErrExist) {
return err return err
} }
} }
if err := os.Mkdir(path.Join(mod, "pkg"), 0755); err != nil { if err := os.Mkdir(path.Join(modDir, "pkg"), 0755); err != nil {
if !errors.Is(err, os.ErrExist) { if !errors.Is(err, os.ErrExist) {
return err return err
} }
@ -395,6 +396,8 @@ func vendorUniverse(ctx context.Context, p string) error {
} }
log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe") log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe")
// FIXME(samalba): disabled install remote stdlib temporarily
// if err := mod.InstallStdlib(p); err != nil {
if err := stdlib.Vendor(ctx, p); err != nil { if err := stdlib.Vendor(ctx, p); err != nil {
return err return err
} }

21
state/state.go
View File

@ -35,8 +35,8 @@ type State struct {
func (s *State) CompilePlan(ctx context.Context) (*compiler.Value, error) { func (s *State) CompilePlan(ctx context.Context) (*compiler.Value, error) {
w := s.Project w := s.Project
// FIXME: backward compatibility // FIXME: backward compatibility
if mod := s.Plan.Module; mod != "" { if planModule := s.Plan.Module; planModule != "" {
w = path.Join(w, mod) w = path.Join(w, planModule)
} }
// FIXME: universe vendoring // FIXME: universe vendoring
@ -51,7 +51,7 @@ func (s *State) CompilePlan(ctx context.Context) (*compiler.Value, error) {
return nil, err return nil, err
} }
args := []string{} var args []string
if pkg := s.Plan.Package; pkg != "" { if pkg := s.Plan.Package; pkg != "" {
args = append(args, pkg) args = append(args, pkg)
} }
@ -81,21 +81,6 @@ func (s *State) CompileInputs() (*compiler.Value, error) {
return v, nil return v, nil
} }
// VendorUniverse vendors the latest (built-in) version of the universe into the
// environment's `cue.mod`.
// FIXME: This has nothing to do in `State` and should be tied to a `Project`.
// However, since environments could point to different modules before, we have
// to handle vendoring on a per environment basis.
func (s *State) VendorUniverse(ctx context.Context) error {
w := s.Project
// FIXME: backward compatibility
if mod := s.Plan.Module; mod != "" {
w = path.Join(w, mod)
}
return vendorUniverse(ctx, w)
}
type Plan struct { type Plan struct {
Module string `yaml:"module,omitempty"` Module string `yaml:"module,omitempty"`
Package string `yaml:"package,omitempty"` Package string `yaml:"package,omitempty"`

1
state/state_test.go Normal file
View File

@ -0,0 +1 @@
package state