kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Merge pull request #1076 from samalba/remote-stdlib

Browse Source 
Support version constraints on remote modules
This commit is contained in:
Sam Alba 2021-10-26 14:54:52 -07:00 committed by GitHub
commit fa334f4a86
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 174 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/dagger/cmd/mod/get.go
View File

@ -38,13 +38,13 @@ var getCmd = &cobra.Command{
var err error
if update && len(args) == 0 {
lg.Info().Msg("updating all installed packages...")
processedRequires, err = mod.UpdateInstalled(project.Path)
processedRequires, err = mod.UpdateInstalled(ctx, project.Path)
} else if update && len(args) > 0 {
lg.Info().Msg("updating specified packages...")
processedRequires, err = mod.UpdateAll(project.Path, args)
processedRequires, err = mod.UpdateAll(ctx, project.Path, args)
} else if !update && len(args) > 0 {
lg.Info().Msg("installing specified packages...")
processedRequires, err = mod.InstallAll(project.Path, args)
processedRequires, err = mod.InstallAll(ctx, project.Path, args)
} else {
lg.Fatal().Msg("unrecognized update/install operation")
}

31
mod/file.go
View File

@ -2,6 +2,7 @@ package mod
import (
"bytes"
"context"
"errors"
"fmt"
"io"
@ -15,11 +16,13 @@ import (
"github.com/spf13/viper"
)
const modFilePath = "./cue.mod/dagger.mod"
const sumFilePath = "./cue.mod/dagger.sum"
const lockFilePath = "./cue.mod/dagger.lock"
const destBasePath = "./cue.mod/pkg"
const tmpBasePath = "./cue.mod/tmp"
const (
modFilePath = "./cue.mod/dagger.mod"
sumFilePath = "./cue.mod/dagger.sum"
lockFilePath = "./cue.mod/dagger.lock"
destBasePath = "./cue.mod/pkg"
tmpBasePath = "./cue.mod/tmp"
)
// file is the parsed, interpreted form of dagger.mod file.
type file struct {
@ -98,7 +101,8 @@ func read(fMod, fSum io.Reader) (*file, error) {
return nil, fmt.Errorf("repos in mod and sum line don't match: %s and %s", modSplit[0], sumSplit[0])
}
require, err := newRequire(modSplit[0])
// FIXME: if we want to add support for version constraints in the mod file, it would be here
require, err := newRequire(modSplit[0], "")
if err != nil {
return nil, err
}
@ -126,20 +130,19 @@ func nonEmptyLines(b []byte) []string {
}
trimmed = spaceRgx.ReplaceAllString(trimmed, " ")
lines = append(lines, trimmed)
}
return lines
}
func (f *file) install(req *Require) error {
func (f *file) install(ctx context.Context, req *Require) error {
// cleaning up possible leftovers
tmpPath := path.Join(f.workspacePath, tmpBasePath, req.fullPath())
defer os.RemoveAll(tmpPath)
// clone to a tmp directory
r, err := clone(req, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password"))
r, err := clone(ctx, req, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password"))
if err != nil {
return fmt.Errorf("error downloading package %s: %w", req, err)
}
@ -168,7 +171,7 @@ func (f *file) install(req *Require) error {
// checkout the cloned repo to that tag, change the version in the existing requirement and
// replace the code in the /pkg folder
existing.version = req.version
if err = r.checkout(req.version); err != nil {
if err = r.checkout(ctx, req.version); err != nil {
return err
}
@ -186,7 +189,7 @@ func (f *file) install(req *Require) error {
return nil
}
func (f *file) updateToLatest(req *Require) (*Require, error) {
func (f *file) updateToLatest(ctx context.Context, req *Require) (*Require, error) {
// check if it doesn't exist
existing := f.searchInstalledRequire(req)
if existing == nil {
@ -198,13 +201,13 @@ func (f *file) updateToLatest(req *Require) (*Require, error) {
defer os.RemoveAll(tmpPath)
// clone to a tmp directory
gitRepo, err := clone(existing, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password"))
gitRepo, err := clone(ctx, existing, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password"))
if err != nil {
return nil, fmt.Errorf("error downloading package %s: %w", existing, err)
}
// checkout the latest tag
latestTag, err := gitRepo.latestTag()
latestTag, err := gitRepo.latestTag(ctx, req.versionConstraint)
if err != nil {
return nil, err
}
@ -219,7 +222,7 @@ func (f *file) updateToLatest(req *Require) (*Require, error) {
}
existing.version = latestTag
if err = gitRepo.checkout(existing.version); err != nil {
if err = gitRepo.checkout(ctx, existing.version); err != nil {
return nil, err
}

51
mod/mod.go
View File

@ -1,19 +1,32 @@
package mod
import (
"context"
"path"
"strings"
"github.com/gofrs/flock"
"github.com/rs/zerolog/log"
)
func InstallStdlib(workspace string) error {
_, err := Install(workspace, "alpha.dagger.io@v0.1")
const (
UniverseVersionConstraint = ">= 0.1.0, < 0.2"
)
return err
func isUniverse(repoName string) bool {
return strings.HasPrefix(strings.ToLower(repoName), "alpha.dagger.io")
}
func Install(workspace, repoName string) (*Require, error) {
require, err := newRequire(repoName)
func Install(ctx context.Context, workspace, repoName, versionConstraint string) (*Require, error) {
lg := log.Ctx(ctx)
if isUniverse(repoName) {
// override versionConstraint to lock the version of universe we vendor
versionConstraint = UniverseVersionConstraint
}
lg.Debug().Str("name", repoName).Msg("installing module")
require, err := newRequire(repoName, versionConstraint)
if err != nil {
return nil, err
}
@ -28,7 +41,7 @@ func Install(workspace, repoName string) (*Require, error) {
return nil, err
}
err = modfile.install(require)
err = modfile.install(ctx, require)
if err != nil {
return nil, err
}
@ -44,14 +57,14 @@ func Install(workspace, repoName string) (*Require, error) {
return require, nil
}
func InstallAll(workspace string, repoNames []string) ([]*Require, error) {
func InstallAll(ctx context.Context, workspace string, repoNames []string) ([]*Require, error) {
installedRequires := make([]*Require, 0, len(repoNames))
var err error
for _, repoName := range repoNames {
var require *Require
if require, err = Install(workspace, repoName); err != nil {
if require, err = Install(ctx, workspace, repoName, ""); err != nil {
continue
}
@ -61,8 +74,16 @@ func InstallAll(workspace string, repoNames []string) ([]*Require, error) {
return installedRequires, err
}
func Update(workspace, repoName string) (*Require, error) {
require, err := newRequire(repoName)
func Update(ctx context.Context, workspace, repoName, versionConstraint string) (*Require, error) {
lg := log.Ctx(ctx)
if isUniverse(repoName) {
// override versionConstraint to lock the version of universe we vendor
versionConstraint = UniverseVersionConstraint
}
lg.Debug().Str("name", repoName).Msg("updating module")
require, err := newRequire(repoName, versionConstraint)
if err != nil {
return nil, err
}
@ -77,7 +98,7 @@ func Update(workspace, repoName string) (*Require, error) {
return nil, err
}
updatedRequire, err := modfile.updateToLatest(require)
updatedRequire, err := modfile.updateToLatest(ctx, require)
if err != nil {
return nil, err
}
@ -93,14 +114,14 @@ func Update(workspace, repoName string) (*Require, error) {
return updatedRequire, nil
}
func UpdateAll(workspace string, repoNames []string) ([]*Require, error) {
func UpdateAll(ctx context.Context, workspace string, repoNames []string) ([]*Require, error) {
updatedRequires := make([]*Require, 0, len(repoNames))
var err error
for _, repoName := range repoNames {
var require *Require
if require, err = Update(workspace, repoName); err != nil {
if require, err = Update(ctx, workspace, repoName, ""); err != nil {
continue
}
@ -110,7 +131,7 @@ func UpdateAll(workspace string, repoNames []string) ([]*Require, error) {
return updatedRequires, err
}
func UpdateInstalled(workspace string) ([]*Require, error) {
func UpdateInstalled(ctx context.Context, workspace string) ([]*Require, error) {
modfile, err := readPath(workspace)
if err != nil {
return nil, err
@ -122,7 +143,7 @@ func UpdateInstalled(workspace string) ([]*Require, error) {
repoNames = append(repoNames, require.String())
}
return UpdateAll(workspace, repoNames)
return UpdateAll(ctx, workspace, repoNames)
}
func Ensure(workspace string) error {

71
mod/repo.go
View File

@ -1,12 +1,14 @@
package mod
import (
"context"
"fmt"
"os"
"sort"
"strings"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/rs/zerolog/log"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
@ -15,9 +17,10 @@ import (
type repo struct {
contents *git.Repository
require *Require
}
func clone(require *Require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) {
func clone(ctx context.Context, require *Require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) {
if err := os.RemoveAll(dir); err != nil {
return nil, fmt.Errorf("error cleaning up tmp directory")
}
@ -46,10 +49,11 @@ func clone(require *Require, dir string, privateKeyFile, privateKeyPassword stri
rr := &repo{
contents: r,
require: require,
}
if require.version == "" {
latestTag, err := rr.latestTag()
latestTag, err := rr.latestTag(ctx, require.versionConstraint)
if err != nil {
return nil, err
}
@ -57,19 +61,23 @@ func clone(require *Require, dir string, privateKeyFile, privateKeyPassword stri
require.version = latestTag
}
if err := rr.checkout(require.version); err != nil {
if err := rr.checkout(ctx, require.version); err != nil {
return nil, err
}
return rr, nil
}
func (r *repo) checkout(version string) error {
func (r *repo) checkout(ctx context.Context, version string) error {
lg := log.Ctx(ctx)
h, err := r.contents.ResolveRevision(plumbing.Revision(version))
if err != nil {
return err
}
lg.Debug().Str("repository", r.require.repo).Str("version", version).Str("commit", h.String()).Msg("checkout repo")
w, err := r.contents.Worktree()
if err != nil {
return err
@ -85,25 +93,61 @@ func (r *repo) checkout(version string) error {
return nil
}
func (r *repo) listTags() ([]string, error) {
func (r *repo) listTagVersions(ctx context.Context, versionConstraint string) ([]string, error) {
lg := log.Ctx(ctx).With().
Str("repository", r.require.repo).
Str("versionConstraint", versionConstraint).
Logger()
if versionConstraint == "" {
versionConstraint = ">= 0"
}
constraint, err := version.NewConstraint(versionConstraint)
if err != nil {
return nil, err
}
iter, err := r.contents.Tags()
if err != nil {
return nil, err
}
var tags []string
if err := iter.ForEach(func(ref *plumbing.Reference) error {
tags = append(tags, ref.Name().Short())
err = iter.ForEach(func(ref *plumbing.Reference) error {
tagV := ref.Name().Short()
if !strings.HasPrefix(tagV, "v") {
lg.Debug().Str("tag", tagV).Msg("tag version ignored, wrong format")
return nil
}); err != nil {
}
v, err := version.NewVersion(tagV)
if err != nil {
lg.Debug().Str("tag", tagV).Err(err).Msg("tag version ignored, parsing error")
return nil
}
if constraint.Check(v) {
// Add tag if it matches the version constraint
tags = append(tags, ref.Name().Short())
lg.Debug().Str("tag", tagV).Msg("version added")
} else {
lg.Debug().Str("tag", tagV).Msg("tag version ignored, does not satisfy constraint")
}
return nil
})
if err != nil {
return nil, err
}
return tags, nil
}
func (r *repo) latestTag() (string, error) {
versionsRaw, err := r.listTags()
func (r *repo) latestTag(ctx context.Context, versionConstraint string) (string, error) {
versionsRaw, err := r.listTagVersions(ctx, versionConstraint)
if err != nil {
return "", err
}
@ -115,10 +159,11 @@ func (r *repo) latestTag() (string, error) {
}
if len(versions) == 0 {
return "", fmt.Errorf("repo doesn't have any tags")
return "", fmt.Errorf("repo doesn't have any tags matching the required version")
}
sort.Sort(version.Collection(versions))
sort.Sort(sort.Reverse(version.Collection(versions)))
version := versions[0].Original()
return versions[len(versions)-1].Original(), nil
return version, nil
}

44
mod/repo_test.go
View File

@ -1,6 +1,7 @@
package mod
import (
"context"
"io/ioutil"
"os"
"testing"
@ -50,7 +51,7 @@ func TestClone(t *testing.T) {
defer os.Remove(tmpDir)
_, err = clone(&c.require, tmpDir, c.privateKeyFile, c.privateKeyPassword)
_, err = clone(context.TODO(), &c.require, tmpDir, c.privateKeyFile, c.privateKeyPassword)
if err != nil {
t.Error(err)
}
@ -65,7 +66,9 @@ func TestListTags(t *testing.T) {
}
defer os.Remove(tmpDir)
r, err := clone(&Require{
ctx := context.TODO()
r, err := clone(ctx, &Require{
cloneRepo: "github.com/dagger/universe",
clonePath: "stdlib",
version: "",
@ -74,7 +77,7 @@ func TestListTags(t *testing.T) {
t.Fatal(err)
}
tags, err := r.listTags()
tags, err := r.listTagVersions(ctx, "")
if err != nil {
t.Error(err)
}
@ -83,3 +86,38 @@ func TestListTags(t *testing.T) {
t.Errorf("could not list repo tags")
}
}
func TestVersionConstraint(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "clone")
if err != nil {
t.Fatal("error creating tmp dir")
}
defer os.Remove(tmpDir)
ctx := context.TODO()
r, err := clone(ctx, &Require{
cloneRepo: "github.com/dagger/universe",
clonePath: "stdlib",
version: "",
}, tmpDir, "", "")
if err != nil {
t.Fatal(err)
}
tagVersion, err := r.latestTag(ctx, "<= 0.1.0")
if err != nil {
t.Error(err)
}
// Make sure we select the right version based on constraint
if tagVersion != "v0.1.0" {
t.Errorf("wrong version: expected 0.1.0, got %v", tagVersion)
}
// Make sure an invalid constraint (version out of range) returns an error
_, err = r.latestTag(ctx, "> 99999")
if err == nil {
t.Error("selected wrong version based on constraint")
}
}

13
mod/require.go
View File

@ -16,15 +16,16 @@ type Require struct {
clonePath string
version string
versionConstraint string
checksum string
}
func newRequire(repoName string) (*Require, error) {
func newRequire(repoName, versionConstraint string) (*Require, error) {
switch {
case strings.HasPrefix(repoName, "github.com"):
return parseGithubRepoName(repoName)
return parseGithubRepoName(repoName, versionConstraint)
case strings.HasPrefix(repoName, "alpha.dagger.io"):
return parseDaggerRepoName(repoName)
return parseDaggerRepoName(repoName, versionConstraint)
default:
return nil, fmt.Errorf("repo name does not match suported providers")
}
@ -32,7 +33,7 @@ func newRequire(repoName string) (*Require, error) {
var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseGithubRepoName(repoName string) (*Require, error) {
func parseGithubRepoName(repoName, versionConstraint string) (*Require, error) {
repoMatches := githubRepoNameRegex.FindStringSubmatch(repoName)
if len(repoMatches) < 4 {
@ -43,6 +44,7 @@ func parseGithubRepoName(repoName string) (*Require, error) {
repo: repoMatches[1],
path: repoMatches[2],
version: repoMatches[3],
versionConstraint: versionConstraint,
cloneRepo: repoMatches[1],
clonePath: repoMatches[2],
@ -51,7 +53,7 @@ func parseGithubRepoName(repoName string) (*Require, error) {
var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseDaggerRepoName(repoName string) (*Require, error) {
func parseDaggerRepoName(repoName, versionConstraint string) (*Require, error) {
repoMatches := daggerRepoNameRegex.FindStringSubmatch(repoName)
if len(repoMatches) < 3 {
@ -62,6 +64,7 @@ func parseDaggerRepoName(repoName string) (*Require, error) {
repo: "alpha.dagger.io",
path: repoMatches[1],
version: repoMatches[2],
versionConstraint: versionConstraint,
cloneRepo: "github.com/dagger/universe",
clonePath: path.Join("/stdlib", repoMatches[1]),

2
mod/require_test.go
View File

@ -98,7 +98,7 @@ func TestParseArgument(t *testing.T) {
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := newRequire(c.in)
got, err := newRequire(c.in, "")
if err != nil && c.hasError {
return
}

4
state/project.go
View File

@ -396,9 +396,9 @@ func vendorUniverse(ctx context.Context, p string) error {
}
log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe")
// FIXME(samalba): disabled install remote stdlib temporarily
// if err := mod.InstallStdlib(p); err != nil {
if err := stdlib.Vendor(ctx, p); err != nil {
// FIXME(samalba): disabled install remote stdlib temporarily
// if _, err := mod.Install(ctx, p, "alpha.dagger.io", ""); err != nil {
return err
}