kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Vendoring improved

Browse Source 
* update dagger init with package manager downloading stdlib

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* split mod get and update functions

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* write to package checksum to dagger.sum when installing/updating

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* encure checksums are valid when compiling input

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* remove references to github.com/tjovicic in docs 1010 and 1011

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* refactor mod get command

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* update logic of moving dir when installing packages

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* fix linting errors

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* revert changes to 1010 docs

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* updating error log line in mod/get

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* fix ci tests when using vendoring

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* update alpha.dagger.io version to v0.1

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* fix mod repo test

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* return error if package already installed

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* remove already installed packages when installing

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* fix issue when vendoring stdlib

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* update mod command with filelock while installing

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* fix linting errors

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

* fix path of mod lock file

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

Co-authored-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

Signed-off-by: Sam Alba <sam.alba@gmail.com>
This commit is contained in:
Sam Alba
2021-10-13 14:32:06 -07:00
committed by Sam Alba
parent 31255f75ad
commit e9ca8f38e6
22 changed files with 709 additions and 428 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
cmd/dagger/cmd/mod/arg.go
View File

@@ -1,57 +0,0 @@
package mod
import (
"fmt"
"path"
"regexp"
"strings"
)
func parseArgument(arg string) (*require, error) {
switch {
case strings.HasPrefix(arg, "github.com"):
return parseGithubRepoName(arg)
case strings.HasPrefix(arg, "alpha.dagger.io"):
return parseDaggerRepoName(arg)
default:
return nil, fmt.Errorf("repo name does not match suported providers")
}
}
var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseGithubRepoName(arg string) (*require, error) {
repoMatches := githubRepoNameRegex.FindStringSubmatch(arg)
if len(repoMatches) < 4 {
return nil, fmt.Errorf("issue when parsing github repo")
}
return &require{
repo: repoMatches[1],
path: repoMatches[2],
version: repoMatches[3],
cloneRepo: repoMatches[1],
clonePath: repoMatches[2],
}, nil
}
var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseDaggerRepoName(arg string) (*require, error) {
repoMatches := daggerRepoNameRegex.FindStringSubmatch(arg)
if len(repoMatches) < 3 {
return nil, fmt.Errorf("issue when parsing dagger repo")
}
return &require{
repo: "alpha.dagger.io",
path: repoMatches[1],
version: repoMatches[2],
cloneRepo: "github.com/dagger/dagger",
clonePath: path.Join("/stdlib", repoMatches[1]),
}, nil
}

100
cmd/dagger/cmd/mod/arg_test.go
View File

@@ -1,100 +0,0 @@
package mod
import "testing"
func TestParseArgument(t *testing.T) {
cases := []struct {
name string
in string
want *require
hasError bool
}{
{
name: "Random",
in: "abcd/bla@:/xyz",
hasError: true,
},
{
name: "Dagger repo",
in: "github.com/dagger/dagger",
want: &require{
repo: "github.com/dagger/dagger",
path: "",
version: "",
},
},
{
name: "Dagger repo with path",
in: "github.com/dagger/dagger/stdlib",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib",
version: "",
},
},
{
name: "Dagger repo with longer path",
in: "github.com/dagger/dagger/stdlib/test/test",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib/test/test",
version: "",
},
},
{
name: "Dagger repo with path and version",
in: "github.com/dagger/dagger/stdlib@v0.1",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib",
version: "v0.1",
},
},
{
name: "Dagger repo with longer path and version",
in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib/test/test",
version: "v0.0.1",
},
},
{
name: "Alpha Dagger repo",
in: "alpha.dagger.io/gcp/gke@v0.1.0-alpha.20",
want: &require{
repo: "alpha.dagger.io",
path: "/gcp/gke",
version: "v0.1.0-alpha.20",
cloneRepo: "github.com/dagger/dagger",
clonePath: "/stdlib/gcp/gke",
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := parseArgument(c.in)
if err != nil && c.hasError {
return
}
if err != nil {
t.Fatal(err)
}
if got.repo != c.want.repo {
t.Errorf("repos differ: want %s, got %s", c.want.repo, got.repo)
}
if got.path != c.want.path {
t.Errorf("paths differ: want %s, got %s", c.want.path, got.path)
}
if got.version != c.want.version {
t.Errorf("versions differ: want %s, got %s", c.want.version, got.version)
}
})
}
}

223
cmd/dagger/cmd/mod/file.go
View File

@@ -1,223 +0,0 @@
package mod
import (
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"io/ioutil"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"github.com/spf13/viper"
)
const filePath = "./cue.mod/dagger.mod"
const destBasePath = "./cue.mod/pkg"
const tmpBasePath = "./cue.mod/tmp"
// A file is the parsed, interpreted form of a cue.mod file.
type file struct {
require []*require
projectPath string
}
func readPath(projectPath string) (*file, error) {
p := path.Join(projectPath, filePath)
f, err := os.Open(p)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return nil, err
}
// dagger.mod.cue doesn't exist, let's create an empty file
if f, err = os.Create(p); err != nil {
return nil, err
}
}
modFile, err := read(f)
if err != nil {
return nil, err
}
modFile.projectPath = projectPath
return modFile, nil
}
func read(f io.Reader) (*file, error) {
b, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
lines := nonEmptyLines(b)
requires := make([]*require, 0, len(lines))
for _, line := range lines {
split := strings.Split(line, " ")
r, err := parseArgument(split[0])
if err != nil {
return nil, err
}
r.version = split[1]
requires = append(requires, r)
}
return &file{
require: requires,
}, nil
}
var spaceRgx = regexp.MustCompile(`\s+`)
func nonEmptyLines(b []byte) []string {
s := strings.ReplaceAll(string(b), "\r\n", "\n")
split := strings.Split(s, "\n")
lines := make([]string, 0, len(split))
for _, l := range split {
trimmed := strings.TrimSpace(l)
if trimmed == "" {
continue
}
trimmed = spaceRgx.ReplaceAllString(trimmed, " ")
lines = append(lines, trimmed)
}
return lines
}
func (f *file) processRequire(req *require, upgrade bool) (bool, error) {
var isNew bool
tmpPath := path.Join(f.projectPath, tmpBasePath, req.repo)
if err := os.MkdirAll(tmpPath, 0755); err != nil {
return false, fmt.Errorf("error creating tmp dir for cloning package")
}
defer os.RemoveAll(tmpPath)
// clone the repo
privateKeyFile := viper.GetString("private-key-file")
privateKeyPassword := viper.GetString("private-key-password")
r, err := clone(req, tmpPath, privateKeyFile, privateKeyPassword)
if err != nil {
return isNew, fmt.Errorf("error downloading package %s: %w", req, err)
}
existing := f.search(req)
destPath := path.Join(f.projectPath, destBasePath)
// requirement is new, so we should move the files and add it to the mod file
if existing == nil {
if err := move(req, tmpPath, destPath); err != nil {
return isNew, err
}
f.require = append(f.require, req)
isNew = true
return isNew, nil
}
if upgrade {
latestTag, err := r.latestTag()
if err != nil {
return isNew, err
}
if latestTag == "" {
return isNew, fmt.Errorf("repo does not have a tag")
}
req.version = latestTag
}
c, err := compareVersions(existing.version, req.version)
if err != nil {
return isNew, err
}
// the existing requirement is newer or equal so we skip installation
if c >= 0 {
return isNew, nil
}
// the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing
// requirement and replace the code in the /pkg folder
existing.version = req.version
if err = r.checkout(req.version); err != nil {
return isNew, err
}
if err = replace(req, tmpPath, destPath); err != nil {
return isNew, err
}
isNew = true
return isNew, nil
}
func (f *file) write() error {
return ioutil.WriteFile(path.Join(f.projectPath, filePath), f.contents().Bytes(), 0600)
}
func (f *file) contents() *bytes.Buffer {
var b bytes.Buffer
for _, r := range f.require {
b.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.version))
}
return &b
}
func (f *file) search(r *require) *require {
for _, existing := range f.require {
if existing.fullPath() == r.fullPath() {
return existing
}
}
return nil
}
type require struct {
repo string
path string
version string
cloneRepo string
clonePath string
}
func (r *require) fullPath() string {
return path.Join(r.repo, r.path)
}
func move(r *require, sourceRepoPath, destBasePath string) error {
destPath := path.Join(destBasePath, r.fullPath())
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
return err
}
if err := os.Rename(path.Join(sourceRepoPath, r.path), destPath); err != nil {
return err
}
return nil
}
func replace(r *require, sourceRepoPath, destBasePath string) error {
if err := os.RemoveAll(path.Join(destBasePath, r.fullPath())); err != nil {
return err
}
return move(r, sourceRepoPath, destBasePath)
}

49
cmd/dagger/cmd/mod/file_test.go
View File

@@ -1,49 +0,0 @@
package mod
import (
"strings"
"testing"
)
func TestReadFile(t *testing.T) {
cases := []struct {
name string
input string
want *file
}{
{
name: "module file with valid dependencies",
input: `
github.com/tjovicic/test xyz
github.com/bla/bla abc
`,
want: &file{
require: []*require{
{
repo: "github.com/tjovicic/test",
path: "",
version: "xyz",
},
{
repo: "github.com/bla/bla",
path: "",
version: "abc",
},
},
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := read(strings.NewReader(c.input))
if err != nil {
t.Error(err)
}
if len(got.require) != len(c.want.require) {
t.Errorf("requires length differs: want %d, got %d", len(c.want.require), len(got.require))
}
})
}
}

80
cmd/dagger/cmd/mod/get.go
View File

@@ -1,11 +1,11 @@
package mod
import (
"github.com/hashicorp/go-version"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"go.dagger.io/dagger/cmd/dagger/cmd/common"
"go.dagger.io/dagger/cmd/dagger/logger"
"go.dagger.io/dagger/mod"
"go.dagger.io/dagger/telemetry"
)
@@ -32,77 +32,41 @@ var getCmd = &cobra.Command{
Value: args,
})
// read mod file in the current dir
modFile, err := readPath(project.Path)
if err != nil {
lg.Fatal().Err(err).Msg("error loading module file")
}
var update = viper.GetBool("update")
// parse packages to install
var packages []*require
var upgrade bool
if len(args) == 0 {
lg.Info().Msg("upgrading installed packages...")
packages = modFile.require
upgrade = true
var processedRequires []*mod.Require
var err error
if update && len(args) == 0 {
lg.Info().Msg("updating all installed packages...")
processedRequires, err = mod.UpdateInstalled(project.Path)
} else if update && len(args) > 0 {
lg.Info().Msg("updating specified packages...")
processedRequires, err = mod.UpdateAll(project.Path, args)
} else if !update && len(args) > 0 {
lg.Info().Msg("installing specified packages...")
processedRequires, err = mod.InstallAll(project.Path, args)
} else {
for _, arg := range args {
p, err := parseArgument(arg)
if err != nil {
lg.Error().Err(err).Msgf("error parsing package %s", arg)
continue
}
packages = append(packages, p)
lg.Fatal().Msg("unrecognized update/install operation")
}
if len(processedRequires) > 0 {
for _, r := range processedRequires {
lg.Info().Msgf("installed/updated package %s", r)
}
}
// download packages
for _, p := range packages {
isNew, err := modFile.processRequire(p, upgrade)
if err != nil {
lg.Error().Err(err).Msgf("error processing package %s", p.repo)
}
if isNew {
lg.Info().Msgf("downloading %s:%v", p.repo, p.version)
}
}
// write to mod file in the current dir
if err = modFile.write(); err != nil {
lg.Error().Err(err).Msg("error writing to mod file")
if err != nil {
lg.Error().Err(err).Msg("error installing/updating packages")
}
<-doneCh
},
}
func compareVersions(reqV1, reqV2 string) (int, error) {
v1, err := version.NewVersion(reqV1)
if err != nil {
return 0, err
}
v2, err := version.NewVersion(reqV2)
if err != nil {
return 0, err
}
if v1.LessThan(v2) {
return -1, nil
}
if v1.Equal(v2) {
return 0, nil
}
return 1, nil
}
func init() {
getCmd.Flags().String("private-key-file", "", "Private ssh key")
getCmd.Flags().String("private-key-password", "", "Private ssh key password")
getCmd.Flags().BoolP("update", "u", false, "Update specified package")
if err := viper.BindPFlags(getCmd.Flags()); err != nil {
panic(err)

128
cmd/dagger/cmd/mod/repo.go
View File

@@ -1,128 +0,0 @@
package mod
import (
"fmt"
"os"
"path"
"sort"
"strings"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/hashicorp/go-version"
)
type repo struct {
localPath string
contents *git.Repository
}
func clone(require *require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) {
o := git.CloneOptions{
URL: fmt.Sprintf("https://%s", require.cloneRepo),
}
if privateKeyFile != "" {
publicKeys, err := ssh.NewPublicKeysFromFile("git", privateKeyFile, privateKeyPassword)
if err != nil {
return nil, err
}
o.Auth = publicKeys
o.URL = fmt.Sprintf("git@%s", strings.Replace(require.cloneRepo, "/", ":", 1))
}
r, err := git.PlainClone(dir, false, &o)
if err != nil {
return nil, err
}
rr := &repo{
localPath: dir,
contents: r,
}
if require.version == "" {
latestTag, err := rr.latestTag()
if err != nil {
return nil, err
}
if latestTag == "" {
return nil, fmt.Errorf("no git tags found in the repo")
}
require.version = latestTag
}
if err := rr.checkout(require.version); err != nil {
return nil, err
}
if _, err := os.Stat(path.Join(dir, require.clonePath, filePath)); err != nil {
return nil, fmt.Errorf("repo does not contain %s", filePath)
}
return rr, nil
}
func (r *repo) checkout(version string) error {
h, err := r.contents.ResolveRevision(plumbing.Revision(version))
if err != nil {
return err
}
w, err := r.contents.Worktree()
if err != nil {
return err
}
err = w.Checkout(&git.CheckoutOptions{
Hash: *h,
})
if err != nil {
return err
}
return nil
}
func (r *repo) listTags() ([]string, error) {
iter, err := r.contents.Tags()
if err != nil {
return nil, err
}
var tags []string
if err := iter.ForEach(func(ref *plumbing.Reference) error {
tags = append(tags, ref.Name().Short())
return nil
}); err != nil {
return nil, err
}
return tags, nil
}
func (r *repo) latestTag() (string, error) {
versionsRaw, err := r.listTags()
if err != nil {
return "", err
}
versions := make([]*version.Version, len(versionsRaw))
for i, raw := range versionsRaw {
v, _ := version.NewVersion(raw)
versions[i] = v
}
sort.Sort(version.Collection(versions))
if len(versions) == 0 {
return "", nil
}
return versions[len(versions)-1].Original(), nil
}

93
cmd/dagger/cmd/mod/repo_test.go
View File

@@ -1,93 +0,0 @@
package mod
import (
"io/ioutil"
"os"
"testing"
)
func TestClone(t *testing.T) {
cases := []struct {
name string
require require
privateKeyFile string
privateKeyPassword string
}{
{
name: "resolving shorter hash version",
require: require{
cloneRepo: "github.com/tjovicic/dagger-modules",
clonePath: "gcpcloudrun",
version: "26a1d46d1b3c",
},
},
{
name: "resolving branch name",
require: require{
cloneRepo: "github.com/tjovicic/dagger-modules",
clonePath: "gcpcloudrun",
version: "main",
},
},
{
name: "resolving tag",
require: require{
cloneRepo: "github.com/tjovicic/dagger-modules",
clonePath: "gcpcloudrun",
version: "v0.3",
},
},
{
name: "Dagger private test repo",
require: require{
cloneRepo: "github.com/dagger/test",
clonePath: "",
version: "main",
},
privateKeyFile: "./test-ssh-keys/id_ed25519_test",
privateKeyPassword: "",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "clone")
if err != nil {
t.Fatal("error creating tmp dir")
}
defer os.Remove(tmpDir)
_, err = clone(&c.require, tmpDir, c.privateKeyFile, c.privateKeyPassword)
if err != nil {
t.Error(err)
}
})
}
}
func TestListTags(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "clone")
if err != nil {
t.Fatal("error creating tmp dir")
}
defer os.Remove(tmpDir)
r, err := clone(&require{
cloneRepo: "github.com/tjovicic/dagger-modules",
clonePath: "gcpcloudrun",
version: "",
}, tmpDir, "", "")
if err != nil {
t.Fatal(err)
}
tags, err := r.listTags()
if err != nil {
t.Error(err)
}
if len(tags) == 0 {
t.Errorf("could not list repo tags")
}
}

7
cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test
View File

@@ -1,7 +0,0 @@
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
QyNTUxOQAAACCpGsk8WLx7gXCXX1muGhKjlkqaqykF1X198WQMkBO2pwAAAKC5Ec8WuRHP
FgAAAAtzc2gtZWQyNTUxOQAAACCpGsk8WLx7gXCXX1muGhKjlkqaqykF1X198WQMkBO2pw
AAAEBXE9Uht+QHuyK7+yYcZFVWOJ3qkhUh/wn289nDKDPHKakayTxYvHuBcJdfWa4aEqOW
SpqrKQXVfX3xZAyQE7anAAAAGnRpaG9taXIuam92aWNpY0B0b3B0YWwuY29tAQID
-----END OPENSSH PRIVATE KEY-----

1
cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test.pub
View File

@@ -1 +0,0 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKkayTxYvHuBcJdfWa4aEqOWSpqrKQXVfX3xZAyQE7an tihomir.jovicic@toptal.com