kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Merge pull request #934 from tjovicic/vendoring-improved

Browse Source 
Use package manager when vendoring stdlib
This commit is contained in:
Sam Alba 2021-10-13 20:30:17 -07:00 committed by GitHub
commit 63bb368d08
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 704 additions and 423 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -27,3 +27,7 @@
tests/node_modules
dist/
docs/learn/tests/node_modules
# Integration CI dagger.mod, dagger.sum
dagger.mod
dagger.sum

57
cmd/dagger/cmd/mod/arg.go
View File

@ -1,57 +0,0 @@
package mod
import (
"fmt"
"path"
"regexp"
"strings"
)
func parseArgument(arg string) (*require, error) {
switch {
case strings.HasPrefix(arg, "github.com"):
return parseGithubRepoName(arg)
case strings.HasPrefix(arg, "alpha.dagger.io"):
return parseDaggerRepoName(arg)
default:
return nil, fmt.Errorf("repo name does not match suported providers")
}
}
var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseGithubRepoName(arg string) (*require, error) {
repoMatches := githubRepoNameRegex.FindStringSubmatch(arg)
if len(repoMatches) < 4 {
return nil, fmt.Errorf("issue when parsing github repo")
}
return &require{
repo: repoMatches[1],
path: repoMatches[2],
version: repoMatches[3],
cloneRepo: repoMatches[1],
clonePath: repoMatches[2],
}, nil
}
var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseDaggerRepoName(arg string) (*require, error) {
repoMatches := daggerRepoNameRegex.FindStringSubmatch(arg)
if len(repoMatches) < 3 {
return nil, fmt.Errorf("issue when parsing dagger repo")
}
return &require{
repo: "alpha.dagger.io",
path: repoMatches[1],
version: repoMatches[2],
cloneRepo: "github.com/dagger/dagger",
clonePath: path.Join("/stdlib", repoMatches[1]),
}, nil
}

223
cmd/dagger/cmd/mod/file.go
View File

@ -1,223 +0,0 @@
package mod
import (
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"io/ioutil"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"github.com/spf13/viper"
)
const filePath = "./cue.mod/dagger.mod"
const destBasePath = "./cue.mod/pkg"
const tmpBasePath = "./cue.mod/tmp"
// A file is the parsed, interpreted form of a cue.mod file.
type file struct {
require []*require
projectPath string
}
func readPath(projectPath string) (*file, error) {
p := path.Join(projectPath, filePath)
f, err := os.Open(p)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return nil, err
}
// dagger.mod.cue doesn't exist, let's create an empty file
if f, err = os.Create(p); err != nil {
return nil, err
}
}
modFile, err := read(f)
if err != nil {
return nil, err
}
modFile.projectPath = projectPath
return modFile, nil
}
func read(f io.Reader) (*file, error) {
b, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
lines := nonEmptyLines(b)
requires := make([]*require, 0, len(lines))
for _, line := range lines {
split := strings.Split(line, " ")
r, err := parseArgument(split[0])
if err != nil {
return nil, err
}
r.version = split[1]
requires = append(requires, r)
}
return &file{
require: requires,
}, nil
}
var spaceRgx = regexp.MustCompile(`\s+`)
func nonEmptyLines(b []byte) []string {
s := strings.ReplaceAll(string(b), "\r\n", "\n")
split := strings.Split(s, "\n")
lines := make([]string, 0, len(split))
for _, l := range split {
trimmed := strings.TrimSpace(l)
if trimmed == "" {
continue
}
trimmed = spaceRgx.ReplaceAllString(trimmed, " ")
lines = append(lines, trimmed)
}
return lines
}
func (f *file) processRequire(req *require, upgrade bool) (bool, error) {
var isNew bool
tmpPath := path.Join(f.projectPath, tmpBasePath, req.repo)
if err := os.MkdirAll(tmpPath, 0755); err != nil {
return false, fmt.Errorf("error creating tmp dir for cloning package")
}
defer os.RemoveAll(tmpPath)
// clone the repo
privateKeyFile := viper.GetString("private-key-file")
privateKeyPassword := viper.GetString("private-key-password")
r, err := clone(req, tmpPath, privateKeyFile, privateKeyPassword)
if err != nil {
return isNew, fmt.Errorf("error downloading package %s: %w", req, err)
}
existing := f.search(req)
destPath := path.Join(f.projectPath, destBasePath)
// requirement is new, so we should move the files and add it to the mod file
if existing == nil {
if err := move(req, tmpPath, destPath); err != nil {
return isNew, err
}
f.require = append(f.require, req)
isNew = true
return isNew, nil
}
if upgrade {
latestTag, err := r.latestTag()
if err != nil {
return isNew, err
}
if latestTag == "" {
return isNew, fmt.Errorf("repo does not have a tag")
}
req.version = latestTag
}
c, err := compareVersions(existing.version, req.version)
if err != nil {
return isNew, err
}
// the existing requirement is newer or equal so we skip installation
if c >= 0 {
return isNew, nil
}
// the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing
// requirement and replace the code in the /pkg folder
existing.version = req.version
if err = r.checkout(req.version); err != nil {
return isNew, err
}
if err = replace(req, tmpPath, destPath); err != nil {
return isNew, err
}
isNew = true
return isNew, nil
}
func (f *file) write() error {
return ioutil.WriteFile(path.Join(f.projectPath, filePath), f.contents().Bytes(), 0600)
}
func (f *file) contents() *bytes.Buffer {
var b bytes.Buffer
for _, r := range f.require {
b.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.version))
}
return &b
}
func (f *file) search(r *require) *require {
for _, existing := range f.require {
if existing.fullPath() == r.fullPath() {
return existing
}
}
return nil
}
type require struct {
repo string
path string
version string
cloneRepo string
clonePath string
}
func (r *require) fullPath() string {
return path.Join(r.repo, r.path)
}
func move(r *require, sourceRepoPath, destBasePath string) error {
destPath := path.Join(destBasePath, r.fullPath())
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
return err
}
if err := os.Rename(path.Join(sourceRepoPath, r.path), destPath); err != nil {
return err
}
return nil
}
func replace(r *require, sourceRepoPath, destBasePath string) error {
if err := os.RemoveAll(path.Join(destBasePath, r.fullPath())); err != nil {
return err
}
return move(r, sourceRepoPath, destBasePath)
}

80
cmd/dagger/cmd/mod/get.go
View File

@ -1,11 +1,11 @@
package mod
import (
"github.com/hashicorp/go-version"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"go.dagger.io/dagger/cmd/dagger/cmd/common"
"go.dagger.io/dagger/cmd/dagger/logger"
"go.dagger.io/dagger/mod"
"go.dagger.io/dagger/telemetry"
)
@ -32,77 +32,41 @@ var getCmd = &cobra.Command{
Value: args,
})
// read mod file in the current dir
modFile, err := readPath(project.Path)
if err != nil {
lg.Fatal().Err(err).Msg("error loading module file")
}
var update = viper.GetBool("update")
// parse packages to install
var packages []*require
var upgrade bool
if len(args) == 0 {
lg.Info().Msg("upgrading installed packages...")
packages = modFile.require
upgrade = true
var processedRequires []*mod.Require
var err error
if update && len(args) == 0 {
lg.Info().Msg("updating all installed packages...")
processedRequires, err = mod.UpdateInstalled(project.Path)
} else if update && len(args) > 0 {
lg.Info().Msg("updating specified packages...")
processedRequires, err = mod.UpdateAll(project.Path, args)
} else if !update && len(args) > 0 {
lg.Info().Msg("installing specified packages...")
processedRequires, err = mod.InstallAll(project.Path, args)
} else {
for _, arg := range args {
p, err := parseArgument(arg)
if err != nil {
lg.Error().Err(err).Msgf("error parsing package %s", arg)
continue
}
packages = append(packages, p)
lg.Fatal().Msg("unrecognized update/install operation")
}
if len(processedRequires) > 0 {
for _, r := range processedRequires {
lg.Info().Msgf("installed/updated package %s", r)
}
}
// download packages
for _, p := range packages {
isNew, err := modFile.processRequire(p, upgrade)
if err != nil {
lg.Error().Err(err).Msgf("error processing package %s", p.repo)
}
if isNew {
lg.Info().Msgf("downloading %s:%v", p.repo, p.version)
}
}
// write to mod file in the current dir
if err = modFile.write(); err != nil {
lg.Error().Err(err).Msg("error writing to mod file")
if err != nil {
lg.Error().Err(err).Msg("error installing/updating packages")
}
<-doneCh
},
}
func compareVersions(reqV1, reqV2 string) (int, error) {
v1, err := version.NewVersion(reqV1)
if err != nil {
return 0, err
}
v2, err := version.NewVersion(reqV2)
if err != nil {
return 0, err
}
if v1.LessThan(v2) {
return -1, nil
}
if v1.Equal(v2) {
return 0, nil
}
return 1, nil
}
func init() {
getCmd.Flags().String("private-key-file", "", "Private ssh key")
getCmd.Flags().String("private-key-password", "", "Private ssh key password")
getCmd.Flags().BoolP("update", "u", false, "Update specified package")
if err := viper.BindPFlags(getCmd.Flags()); err != nil {
panic(err)

8
docs/learn/1010-dev-cue-package.md
View File

@ -42,13 +42,13 @@ domain name (as in Go) followed by a descriptive name. In this example, we reuse
package from it.
```shell
mkdir -p cue.mod/pkg/github.com/tjovicic/gcpcloudrun
mkdir -p cue.mod/pkg/github.com/username/gcpcloudrun
```
Let's write the package logic. It is basically what we've seen in the 106-cloudrun example:
```shell
touch cue.mod/pkg/github.com/tjovicic/gcpcloudrun/source.cue
touch cue.mod/pkg/github.com/username/gcpcloudrun/source.cue
```
```cue file=./tests/dev-cue-package/source.cue title="cue.mod/pkg/github.com/tjovicic/gcpcloudrun/source.cue"
@ -86,8 +86,8 @@ You've probably guessed this package isn't tied to just your project. You can ea
of different projects and use it as we've shown above.
```shell
mkdir -p /my-new-project/cue.mod/pkg/github.com/tjovicic/gcpcloudrun
cp ./cue.mod/pkg/github.com/tjovicic/gcpcloudrun/source.cue /new-project/cue.mod/pkg/github.com/tjovicic/gcpcloudrun
mkdir -p /my-new-workspace/cue.mod/pkg/github.com/username/gcpcloudrun
cp ./cue.mod/pkg/github.com/username/gcpcloudrun/source.cue /new-workspace/cue.mod/pkg/github.com/username/gcpcloudrun
```
## Contributing to Dagger stdlib

4
docs/learn/1011-package-manager.md
View File

@ -59,7 +59,7 @@ dagger mod get github.com/dagger/packages/gcpcloudrun@v0.1
```
It should pull the `v0.1` version from Github, leave a copy in `cue.mod/pkg` and reflect the change in
`cue.mod/dagger.mod.cue` file:
`cue.mod/dagger.mod` file:
```shell
cue.mod/pkg/github.com/
@ -111,7 +111,7 @@ You should see similar output:
12:25PM INF system | downloading github.com/dagger/packages:v0.2
```
And `cue.mod/dagger.mod.cue` should reflect the new version:
And `cue.mod/dagger.mod` should reflect the new version:
```cue title="./cue.mod/dagger.mod"
github.com/dagger/packages/gcpcloudrun v0.2

2
go.mod
View File

@ -11,6 +11,7 @@ require (
github.com/docker/distribution v2.7.1+incompatible
github.com/emicklei/proto v1.9.0 // indirect
github.com/go-git/go-git/v5 v5.4.2
github.com/gofrs/flock v0.8.1
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/google/uuid v1.3.0
github.com/hashicorp/go-version v1.3.0
@ -34,6 +35,7 @@ require (
go.opentelemetry.io/otel/sdk v1.0.1
go.opentelemetry.io/otel/trace v1.0.1
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
golang.org/x/mod v0.4.2
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b

4
go.sum
View File

@ -594,8 +594,9 @@ github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gofrs/flock v0.7.3 h1:I0EKY9l8HZCXTMYC4F80vwT6KNypV9uYKP3Alm/hjmQ=
github.com/gofrs/flock v0.7.3/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
github.com/gogo/googleapis v1.3.2/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
@ -1503,6 +1504,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

31
mod/checksum.go Normal file
View File

@ -0,0 +1,31 @@
package mod
import (
"fmt"
"os"
"path"
"golang.org/x/mod/sumdb/dirhash"
)
func dirChecksum(dirPath string) (string, error) {
err := cleanDirForChecksum(dirPath)
if err != nil {
return "", err
}
checksum, err := dirhash.HashDir(dirPath, "", dirhash.DefaultHash)
if err != nil {
return "", err
}
return checksum, nil
}
func cleanDirForChecksum(dirPath string) error {
if err := os.RemoveAll(path.Join(dirPath, ".git")); err != nil {
return fmt.Errorf("error cleaning up .git directory")
}
return nil
}

293
mod/file.go Normal file
View File

@ -0,0 +1,293 @@
package mod
import (
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"io/ioutil"
"os"
"path"
"regexp"
"strings"
"github.com/spf13/viper"
)
const modFilePath = "./cue.mod/dagger.mod"
const sumFilePath = "./cue.mod/dagger.sum"
const lockFilePath = "./cue.mod/dagger.lock"
const destBasePath = "./cue.mod/pkg"
const tmpBasePath = "./cue.mod/tmp"
// file is the parsed, interpreted form of dagger.mod file.
type file struct {
requires []*Require
workspacePath string
}
func readPath(workspacePath string) (*file, error) {
pMod := path.Join(workspacePath, modFilePath)
fMod, err := os.Open(pMod)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return nil, err
}
// dagger.mod doesn't exist, let's create an empty file
if fMod, err = os.Create(pMod); err != nil {
return nil, err
}
}
pSum := path.Join(workspacePath, sumFilePath)
fSum, err := os.Open(pSum)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return nil, err
}
// dagger.sum doesn't exist, let's create an empty file
if fSum, err = os.Create(pSum); err != nil {
return nil, err
}
}
modFile, err := read(fMod, fSum)
if err != nil {
return nil, err
}
modFile.workspacePath = workspacePath
return modFile, nil
}
func read(fMod, fSum io.Reader) (*file, error) {
bMod, err := ioutil.ReadAll(fMod)
if err != nil {
return nil, err
}
bSum, err := ioutil.ReadAll(fSum)
if err != nil {
return nil, err
}
modLines := nonEmptyLines(bMod)
sumLines := nonEmptyLines(bSum)
if len(modLines) != len(sumLines) {
return nil, fmt.Errorf("length of dagger.mod and dagger.sum files differ")
}
requires := make([]*Require, 0, len(modLines))
for i := 0; i < len(modLines); i++ {
modSplit := strings.Split(modLines[i], " ")
if len(modSplit) != 2 {
return nil, fmt.Errorf("line in the mod file doesn't contain 2 elements")
}
sumSplit := strings.Split(sumLines[i], " ")
if len(sumSplit) != 2 {
return nil, fmt.Errorf("line in the sum file doesn't contain 2 elements")
}
if modSplit[0] != sumSplit[0] {
return nil, fmt.Errorf("repos in mod and sum line don't match: %s and %s", modSplit[0], sumSplit[0])
}
require, err := newRequire(modSplit[0])
if err != nil {
return nil, err
}
require.version = modSplit[1]
require.checksum = sumSplit[1]
requires = append(requires, require)
}
return &file{requires: requires}, nil
}
var spaceRgx = regexp.MustCompile(`\s+`)
func nonEmptyLines(b []byte) []string {
s := strings.ReplaceAll(string(b), "\r\n", "\n")
split := strings.Split(s, "\n")
lines := make([]string, 0, len(split))
for _, l := range split {
trimmed := strings.TrimSpace(l)
if trimmed == "" {
continue
}
trimmed = spaceRgx.ReplaceAllString(trimmed, " ")
lines = append(lines, trimmed)
}
return lines
}
func (f *file) install(req *Require) error {
// cleaning up possible leftovers
tmpPath := path.Join(f.workspacePath, tmpBasePath, req.fullPath())
defer os.RemoveAll(tmpPath)
// clone to a tmp directory
r, err := clone(req, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password"))
if err != nil {
return fmt.Errorf("error downloading package %s: %w", req, err)
}
destPath := path.Join(f.workspacePath, destBasePath, req.fullPath())
// requirement is new, so we should move the cloned files from tmp to pkg and add it to the mod file
existing := f.searchInstalledRequire(req)
if existing == nil {
if err = replace(req, tmpPath, destPath); err != nil {
return err
}
checksum, err := dirChecksum(destPath)
if err != nil {
return err
}
req.checksum = checksum
f.requires = append(f.requires, req)
return nil
}
// checkout the cloned repo to that tag, change the version in the existing requirement and
// replace the code in the /pkg folder
existing.version = req.version
if err = r.checkout(req.version); err != nil {
return err
}
if err = replace(req, tmpPath, destPath); err != nil {
return err
}
checksum, err := dirChecksum(destPath)
if err != nil {
return err
}
existing.checksum = checksum
return nil
}
func (f *file) updateToLatest(req *Require) (*Require, error) {
// check if it doesn't exist
existing := f.searchInstalledRequire(req)
if existing == nil {
return nil, fmt.Errorf("package %s isn't already installed", req.fullPath())
}
// cleaning up possible leftovers
tmpPath := path.Join(f.workspacePath, tmpBasePath, existing.fullPath())
defer os.RemoveAll(tmpPath)
// clone to a tmp directory
gitRepo, err := clone(existing, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password"))
if err != nil {
return nil, fmt.Errorf("error downloading package %s: %w", existing, err)
}
// checkout the latest tag
latestTag, err := gitRepo.latestTag()
if err != nil {
return nil, err
}
c, err := compareVersions(latestTag, existing.version)
if err != nil {
return nil, err
}
if c < 0 {
return nil, fmt.Errorf("latest git tag is less than the current version")
}
existing.version = latestTag
if err = gitRepo.checkout(existing.version); err != nil {
return nil, err
}
// move the package from tmp to pkg directory
destPath := path.Join(f.workspacePath, destBasePath, existing.fullPath())
if err = replace(existing, tmpPath, destPath); err != nil {
return nil, err
}
checksum, err := dirChecksum(destPath)
if err != nil {
return nil, err
}
req.checksum = checksum
return existing, nil
}
func (f *file) searchInstalledRequire(r *Require) *Require {
for _, existing := range f.requires {
if existing.fullPath() == r.fullPath() {
return existing
}
}
return nil
}
func (f *file) ensure() error {
for _, require := range f.requires {
requirePath := path.Join(f.workspacePath, destBasePath, require.fullPath())
checksum, err := dirChecksum(requirePath)
if err != nil {
return nil
}
if require.checksum != checksum {
return fmt.Errorf("wrong checksum for %s", require.fullPath())
}
}
return nil
}
func (f *file) write() error {
// write dagger.mod file
var bMod bytes.Buffer
for _, r := range f.requires {
bMod.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.version))
}
err := ioutil.WriteFile(path.Join(f.workspacePath, modFilePath), bMod.Bytes(), 0600)
if err != nil {
return err
}
// write dagger.sum file
var bSum bytes.Buffer
for _, r := range f.requires {
bSum.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.checksum))
}
err = ioutil.WriteFile(path.Join(f.workspacePath, sumFilePath), bSum.Bytes(), 0600)
if err != nil {
return err
}
return nil
}

21
cmd/dagger/cmd/mod/file_test.go → mod/file_test.go
View File

@ -7,18 +7,23 @@ import (
func TestReadFile(t *testing.T) {
cases := []struct {
name string
input string
want *file
name string
modFile string
sumFile string
want *file
}{
{
name: "module file with valid dependencies",
input: `
modFile: `
github.com/tjovicic/test xyz
github.com/bla/bla abc
`,
sumFile: `
github.com/tjovicic/test h1:hash
github.com/bla/bla h1:hash
`,
want: &file{
require: []*require{
requires: []*Require{
{
repo: "github.com/tjovicic/test",
path: "",
@ -36,13 +41,13 @@ func TestReadFile(t *testing.T) {
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := read(strings.NewReader(c.input))
got, err := read(strings.NewReader(c.modFile), strings.NewReader(c.sumFile))
if err != nil {
t.Error(err)
}
if len(got.require) != len(c.want.require) {
t.Errorf("requires length differs: want %d, got %d", len(c.want.require), len(got.require))
if len(got.requires) != len(c.want.requires) {
t.Errorf("requires length differs: want %d, got %d", len(c.want.requires), len(got.requires))
}
})
}

135
mod/mod.go Normal file
View File

@ -0,0 +1,135 @@
package mod
import (
"path"
"github.com/gofrs/flock"
)
func InstallStdlib(workspace string) error {
_, err := Install(workspace, "alpha.dagger.io@v0.1")
return err
}
func Install(workspace, repoName string) (*Require, error) {
require, err := newRequire(repoName)
if err != nil {
return nil, err
}
modfile, err := readPath(workspace)
if err != nil {
return nil, err
}
fileLock := flock.New(path.Join(workspace, lockFilePath))
if err := fileLock.Lock(); err != nil {
return nil, err
}
err = modfile.install(require)
if err != nil {
return nil, err
}
if err = modfile.write(); err != nil {
return nil, err
}
if err := fileLock.Unlock(); err != nil {
return nil, err
}
return require, nil
}
func InstallAll(workspace string, repoNames []string) ([]*Require, error) {
installedRequires := make([]*Require, 0, len(repoNames))
var err error
for _, repoName := range repoNames {
var require *Require
if require, err = Install(workspace, repoName); err != nil {
continue
}
installedRequires = append(installedRequires, require)
}
return installedRequires, err
}
func Update(workspace, repoName string) (*Require, error) {
require, err := newRequire(repoName)
if err != nil {
return nil, err
}
modfile, err := readPath(workspace)
if err != nil {
return nil, err
}
fileLock := flock.New(path.Join(workspace, lockFilePath))
if err := fileLock.Lock(); err != nil {
return nil, err
}
updatedRequire, err := modfile.updateToLatest(require)
if err != nil {
return nil, err
}
if err = modfile.write(); err != nil {
return nil, err
}
if err := fileLock.Unlock(); err != nil {
return nil, err
}
return updatedRequire, nil
}
func UpdateAll(workspace string, repoNames []string) ([]*Require, error) {
updatedRequires := make([]*Require, 0, len(repoNames))
var err error
for _, repoName := range repoNames {
var require *Require
if require, err = Update(workspace, repoName); err != nil {
continue
}
updatedRequires = append(updatedRequires, require)
}
return updatedRequires, err
}
func UpdateInstalled(workspace string) ([]*Require, error) {
modfile, err := readPath(workspace)
if err != nil {
return nil, err
}
repoNames := make([]string, 0, len(modfile.requires))
for _, require := range modfile.requires {
repoNames = append(repoNames, require.String())
}
return UpdateAll(workspace, repoNames)
}
func Ensure(workspace string) error {
modfile, err := readPath(workspace)
if err != nil {
return err
}
return modfile.ensure()
}

30
cmd/dagger/cmd/mod/repo.go → mod/repo.go
View File

@ -3,7 +3,6 @@ package mod
import (
"fmt"
"os"
"path"
"sort"
"strings"
@ -15,11 +14,17 @@ import (
)
type repo struct {
localPath string
contents *git.Repository
contents *git.Repository
}
func clone(require *require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) {
func clone(require *Require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) {
if err := os.RemoveAll(dir); err != nil {
return nil, fmt.Errorf("error cleaning up tmp directory")
}
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, fmt.Errorf("error creating tmp dir for cloning package")
}
o := git.CloneOptions{
URL: fmt.Sprintf("https://%s", require.cloneRepo),
}
@ -40,8 +45,7 @@ func clone(require *require, dir string, privateKeyFile, privateKeyPassword stri
}
rr := &repo{
localPath: dir,
contents: r,
contents: r,
}
if require.version == "" {
@ -50,10 +54,6 @@ func clone(require *require, dir string, privateKeyFile, privateKeyPassword stri
return nil, err
}
if latestTag == "" {
return nil, fmt.Errorf("no git tags found in the repo")
}
require.version = latestTag
}
@ -61,10 +61,6 @@ func clone(require *require, dir string, privateKeyFile, privateKeyPassword stri
return nil, err
}
if _, err := os.Stat(path.Join(dir, require.clonePath, filePath)); err != nil {
return nil, fmt.Errorf("repo does not contain %s", filePath)
}
return rr, nil
}
@ -118,11 +114,11 @@ func (r *repo) latestTag() (string, error) {
versions[i] = v
}
sort.Sort(version.Collection(versions))
if len(versions) == 0 {
return "", nil
return "", fmt.Errorf("repo doesn't have any tags")
}
sort.Sort(version.Collection(versions))
return versions[len(versions)-1].Original(), nil
}

34
cmd/dagger/cmd/mod/repo_test.go → mod/repo_test.go
View File

@ -9,37 +9,37 @@ import (
func TestClone(t *testing.T) {
cases := []struct {
name string
require require
require Require
privateKeyFile string
privateKeyPassword string
}{
{
name: "resolving shorter hash version",
require: require{
cloneRepo: "github.com/tjovicic/dagger-modules",
clonePath: "gcpcloudrun",
version: "26a1d46d1b3c",
require: Require{
cloneRepo: "github.com/dagger/universe",
clonePath: "stdlib",
version: "24d7af3fc2a3e9c7cc2",
},
},
{
name: "resolving branch name",
require: require{
cloneRepo: "github.com/tjovicic/dagger-modules",
clonePath: "gcpcloudrun",
require: Require{
cloneRepo: "github.com/dagger/universe",
clonePath: "stdlib",
version: "main",
},
},
{
name: "resolving tag",
require: require{
cloneRepo: "github.com/tjovicic/dagger-modules",
clonePath: "gcpcloudrun",
version: "v0.3",
require: Require{
cloneRepo: "github.com/dagger/universe",
clonePath: "stdlib",
version: "v0.1",
},
},
{
name: "Dagger private test repo",
require: require{
name: "dagger private repo",
require: Require{
cloneRepo: "github.com/dagger/test",
clonePath: "",
version: "main",
@ -73,9 +73,9 @@ func TestListTags(t *testing.T) {
}
defer os.Remove(tmpDir)
r, err := clone(&require{
cloneRepo: "github.com/tjovicic/dagger-modules",
clonePath: "gcpcloudrun",
r, err := clone(&Require{
cloneRepo: "github.com/dagger/universe",
clonePath: "stdlib",
version: "",
}, tmpDir, "", "")
if err != nil {

90
mod/require.go Normal file
View File

@ -0,0 +1,90 @@
package mod
import (
"fmt"
"os"
"path"
"regexp"
"strings"
)
type Require struct {
repo string
path string
cloneRepo string
clonePath string
version string
checksum string
}
func newRequire(repoName string) (*Require, error) {
switch {
case strings.HasPrefix(repoName, "github.com"):
return parseGithubRepoName(repoName)
case strings.HasPrefix(repoName, "alpha.dagger.io"):
return parseDaggerRepoName(repoName)
default:
return nil, fmt.Errorf("repo name does not match suported providers")
}
}
var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseGithubRepoName(repoName string) (*Require, error) {
repoMatches := githubRepoNameRegex.FindStringSubmatch(repoName)
if len(repoMatches) < 4 {
return nil, fmt.Errorf("issue when parsing github repo")
}
return &Require{
repo: repoMatches[1],
path: repoMatches[2],
version: repoMatches[3],
cloneRepo: repoMatches[1],
clonePath: repoMatches[2],
}, nil
}
var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseDaggerRepoName(repoName string) (*Require, error) {
repoMatches := daggerRepoNameRegex.FindStringSubmatch(repoName)
if len(repoMatches) < 3 {
return nil, fmt.Errorf("issue when parsing dagger repo")
}
return &Require{
repo: "alpha.dagger.io",
path: repoMatches[1],
version: repoMatches[2],
cloneRepo: "github.com/dagger/universe",
clonePath: path.Join("/stdlib", repoMatches[1]),
}, nil
}
func (r *Require) String() string {
return fmt.Sprintf("%s@%s", r.fullPath(), r.version)
}
func (r *Require) fullPath() string {
return path.Join(r.repo, r.path)
}
func replace(r *Require, sourceRepoPath, destPath string) error {
// remove previous package directory
if err := os.RemoveAll(destPath); err != nil {
return err
}
if err := os.Rename(path.Join(sourceRepoPath, r.clonePath), destPath); err != nil {
return err
}
return nil
}

45
cmd/dagger/cmd/mod/arg_test.go → mod/require_test.go
View File

@ -1,12 +1,14 @@
package mod
import "testing"
import (
"testing"
)
func TestParseArgument(t *testing.T) {
cases := []struct {
name string
in string
want *require
want *Require
hasError bool
}{
{
@ -17,7 +19,7 @@ func TestParseArgument(t *testing.T) {
{
name: "Dagger repo",
in: "github.com/dagger/dagger",
want: &require{
want: &Require{
repo: "github.com/dagger/dagger",
path: "",
version: "",
@ -26,7 +28,7 @@ func TestParseArgument(t *testing.T) {
{
name: "Dagger repo with path",
in: "github.com/dagger/dagger/stdlib",
want: &require{
want: &Require{
repo: "github.com/dagger/dagger",
path: "/stdlib",
version: "",
@ -35,7 +37,7 @@ func TestParseArgument(t *testing.T) {
{
name: "Dagger repo with longer path",
in: "github.com/dagger/dagger/stdlib/test/test",
want: &require{
want: &Require{
repo: "github.com/dagger/dagger",
path: "/stdlib/test/test",
version: "",
@ -44,25 +46,25 @@ func TestParseArgument(t *testing.T) {
{
name: "Dagger repo with path and version",
in: "github.com/dagger/dagger/stdlib@v0.1",
want: &require{
want: &Require{
repo: "github.com/dagger/dagger",
path: "/stdlib",
version: "v0.1",
},
},
{
name: "Dagger repo with longer path and version",
name: "Dagger repo with longer path and version tag",
in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1",
want: &require{
want: &Require{
repo: "github.com/dagger/dagger",
path: "/stdlib/test/test",
version: "v0.0.1",
},
},
{
name: "Alpha Dagger repo",
name: "Alpha Dagger repo with path",
in: "alpha.dagger.io/gcp/gke@v0.1.0-alpha.20",
want: &require{
want: &Require{
repo: "alpha.dagger.io",
path: "/gcp/gke",
version: "v0.1.0-alpha.20",
@ -71,11 +73,32 @@ func TestParseArgument(t *testing.T) {
clonePath: "/stdlib/gcp/gke",
},
},
{
name: "Alpha Dagger repo",
in: "alpha.dagger.io@v0.1.0-alpha.23",
want: &Require{
repo: "alpha.dagger.io",
path: "",
version: "v0.1.0-alpha.23",
cloneRepo: "github.com/dagger/dagger",
clonePath: "/stdlib",
},
},
{
name: "Dagger repo with longer path and commit version",
in: "github.com/dagger/dagger/stdlib/test/test@26a1d46d1b3c",
want: &Require{
repo: "github.com/dagger/dagger",
path: "/stdlib/test/test",
version: "26a1d46d1b3c",
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := parseArgument(c.in)
got, err := newRequire(c.in)
if err != nil && c.hasError {
return
}

0
cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test → mod/test-ssh-keys/id_ed25519_test
View File

0
cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test.pub → mod/test-ssh-keys/id_ed25519_test.pub
View File

27
mod/version.go Normal file
View File

@ -0,0 +1,27 @@
package mod
import "github.com/hashicorp/go-version"
// compareVersions returns -1 if the first argument is less or 1 if it's greater than the second argument.
// It returns 0 if they are equal.
func compareVersions(reqV1, reqV2 string) (int, error) {
v1, err := version.NewVersion(reqV1)
if err != nil {
return 0, err
}
v2, err := version.NewVersion(reqV2)
if err != nil {
return 0, err
}
if v1.LessThan(v2) {
return -1, nil
}
if v1.Equal(v2) {
return 0, nil
}
return 1, nil
}

17
state/project.go
View File

@ -49,6 +49,7 @@ func Init(ctx context.Context, dir string) (*Project, error) {
}
return nil, err
}
if err := os.Mkdir(path.Join(daggerRoot, envDir), 0755); err != nil {
return nil, err
}
@ -342,35 +343,35 @@ func (w *Project) cleanPackageName(ctx context.Context, pkg string) (string, err
return p, nil
}
func cueModInit(ctx context.Context, p string) error {
func cueModInit(ctx context.Context, parentDir string) error {
lg := log.Ctx(ctx)
mod := path.Join(p, "cue.mod")
if err := os.Mkdir(mod, 0755); err != nil {
modDir := path.Join(parentDir, "cue.mod")
if err := os.Mkdir(modDir, 0755); err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
}
modFile := path.Join(mod, "module.cue")
modFile := path.Join(modDir, "module.cue")
if _, err := os.Stat(modFile); err != nil {
if !errors.Is(err, os.ErrNotExist) {
return err
}
lg.Debug().Str("mod", p).Msg("initializing cue.mod")
lg.Debug().Str("mod", parentDir).Msg("initializing cue.mod")
if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil {
return err
}
}
if err := os.Mkdir(path.Join(mod, "usr"), 0755); err != nil {
if err := os.Mkdir(path.Join(modDir, "usr"), 0755); err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
}
if err := os.Mkdir(path.Join(mod, "pkg"), 0755); err != nil {
if err := os.Mkdir(path.Join(modDir, "pkg"), 0755); err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
@ -395,6 +396,8 @@ func vendorUniverse(ctx context.Context, p string) error {
}
log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe")
// FIXME(samalba): disabled install remote stdlib temporarily
// if err := mod.InstallStdlib(p); err != nil {
if err := stdlib.Vendor(ctx, p); err != nil {
return err
}

21
state/state.go
View File

@ -35,8 +35,8 @@ type State struct {
func (s *State) CompilePlan(ctx context.Context) (*compiler.Value, error) {
w := s.Project
// FIXME: backward compatibility
if mod := s.Plan.Module; mod != "" {
w = path.Join(w, mod)
if planModule := s.Plan.Module; planModule != "" {
w = path.Join(w, planModule)
}
// FIXME: universe vendoring
@ -51,7 +51,7 @@ func (s *State) CompilePlan(ctx context.Context) (*compiler.Value, error) {
return nil, err
}
args := []string{}
var args []string
if pkg := s.Plan.Package; pkg != "" {
args = append(args, pkg)
}
@ -81,21 +81,6 @@ func (s *State) CompileInputs() (*compiler.Value, error) {
return v, nil
}
// VendorUniverse vendors the latest (built-in) version of the universe into the
// environment's `cue.mod`.
// FIXME: This has nothing to do in `State` and should be tied to a `Project`.
// However, since environments could point to different modules before, we have
// to handle vendoring on a per environment basis.
func (s *State) VendorUniverse(ctx context.Context) error {
w := s.Project
// FIXME: backward compatibility
if mod := s.Plan.Module; mod != "" {
w = path.Join(w, mod)
}
return vendorUniverse(ctx, w)
}
type Plan struct {
Module string `yaml:"module,omitempty"`
Package string `yaml:"package,omitempty"`

1
state/state_test.go Normal file
View File

@ -0,0 +1 @@
package state