Merge pull request #862 from tjovicic/package-manager

First version of package manager
This commit is contained in:
Sam Alba 2021-08-27 15:25:20 -07:00 committed by GitHub
commit 47ef0a4c2a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 912 additions and 7 deletions

View File

@ -32,7 +32,7 @@ import (
"go.dagger.io/dagger/state"
)
// A dagger client
// Client is a dagger client
type Client struct {
c *bk.Client
cfg Config

57
cmd/dagger/cmd/mod/arg.go Normal file
View File

@ -0,0 +1,57 @@
package mod
import (
"fmt"
"path"
"regexp"
"strings"
)
func parseArgument(arg string) (*require, error) {
switch {
case strings.HasPrefix(arg, "github.com"):
return parseGithubRepoName(arg)
case strings.HasPrefix(arg, "alpha.dagger.io"):
return parseDaggerRepoName(arg)
default:
return nil, fmt.Errorf("repo name does not match suported providers")
}
}
var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseGithubRepoName(arg string) (*require, error) {
repoMatches := githubRepoNameRegex.FindStringSubmatch(arg)
if len(repoMatches) < 4 {
return nil, fmt.Errorf("issue when parsing github repo")
}
return &require{
repo: repoMatches[1],
path: repoMatches[2],
version: repoMatches[3],
cloneRepo: repoMatches[1],
clonePath: repoMatches[2],
}, nil
}
var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseDaggerRepoName(arg string) (*require, error) {
repoMatches := daggerRepoNameRegex.FindStringSubmatch(arg)
if len(repoMatches) < 3 {
return nil, fmt.Errorf("issue when parsing dagger repo")
}
return &require{
repo: "alpha.dagger.io",
path: repoMatches[1],
version: repoMatches[2],
cloneRepo: "github.com/dagger/dagger",
clonePath: path.Join("/stdlib", repoMatches[1]),
}, nil
}

View File

@ -0,0 +1,100 @@
package mod
import "testing"
func TestParseArgument(t *testing.T) {
cases := []struct {
name string
in string
want *require
hasError bool
}{
{
name: "Random",
in: "abcd/bla@:/xyz",
hasError: true,
},
{
name: "Dagger repo",
in: "github.com/dagger/dagger",
want: &require{
repo: "github.com/dagger/dagger",
path: "",
version: "",
},
},
{
name: "Dagger repo with path",
in: "github.com/dagger/dagger/stdlib",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib",
version: "",
},
},
{
name: "Dagger repo with longer path",
in: "github.com/dagger/dagger/stdlib/test/test",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib/test/test",
version: "",
},
},
{
name: "Dagger repo with path and version",
in: "github.com/dagger/dagger/stdlib@v0.1",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib",
version: "v0.1",
},
},
{
name: "Dagger repo with longer path and version",
in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib/test/test",
version: "v0.0.1",
},
},
{
name: "Alpha Dagger repo",
in: "alpha.dagger.io/gcp/gke@v0.1.0-alpha.20",
want: &require{
repo: "alpha.dagger.io",
path: "/gcp/gke",
version: "v0.1.0-alpha.20",
cloneRepo: "github.com/dagger/dagger",
clonePath: "/stdlib/gcp/gke",
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := parseArgument(c.in)
if err != nil && c.hasError {
return
}
if err != nil {
t.Fatal(err)
}
if got.repo != c.want.repo {
t.Errorf("repos differ: want %s, got %s", c.want.repo, got.repo)
}
if got.path != c.want.path {
t.Errorf("paths differ: want %s, got %s", c.want.path, got.path)
}
if got.version != c.want.version {
t.Errorf("versions differ: want %s, got %s", c.want.version, got.version)
}
})
}
}

223
cmd/dagger/cmd/mod/file.go Normal file
View File

@ -0,0 +1,223 @@
package mod
import (
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"io/ioutil"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"github.com/spf13/viper"
)
const filePath = "./cue.mod/dagger.mod"
const destBasePath = "./cue.mod/pkg"
const tmpBasePath = "./cue.mod/tmp"
// A file is the parsed, interpreted form of a cue.mod file.
type file struct {
require []*require
workspacePath string
}
func readPath(workspacePath string) (*file, error) {
p := path.Join(workspacePath, filePath)
f, err := os.Open(p)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return nil, err
}
// dagger.mod.cue doesn't exist, let's create an empty file
if f, err = os.Create(p); err != nil {
return nil, err
}
}
modFile, err := read(f)
if err != nil {
return nil, err
}
modFile.workspacePath = workspacePath
return modFile, nil
}
func read(f io.Reader) (*file, error) {
b, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
lines := nonEmptyLines(b)
requires := make([]*require, 0, len(lines))
for _, line := range lines {
split := strings.Split(line, " ")
r, err := parseArgument(split[0])
if err != nil {
return nil, err
}
r.version = split[1]
requires = append(requires, r)
}
return &file{
require: requires,
}, nil
}
var spaceRgx = regexp.MustCompile(`\s+`)
func nonEmptyLines(b []byte) []string {
s := strings.ReplaceAll(string(b), "\r\n", "\n")
split := strings.Split(s, "\n")
lines := make([]string, 0, len(split))
for _, l := range split {
trimmed := strings.TrimSpace(l)
if trimmed == "" {
continue
}
trimmed = spaceRgx.ReplaceAllString(trimmed, " ")
lines = append(lines, trimmed)
}
return lines
}
func (f *file) processRequire(req *require, upgrade bool) (bool, error) {
var isNew bool
tmpPath := path.Join(f.workspacePath, tmpBasePath, req.repo)
if err := os.MkdirAll(tmpPath, 0755); err != nil {
return false, fmt.Errorf("error creating tmp dir for cloning package")
}
defer os.RemoveAll(tmpPath)
// clone the repo
privateKeyFile := viper.GetString("private-key-file")
privateKeyPassword := viper.GetString("private-key-password")
r, err := clone(req, tmpPath, privateKeyFile, privateKeyPassword)
if err != nil {
return isNew, fmt.Errorf("error downloading package %s: %w", req, err)
}
existing := f.search(req)
destPath := path.Join(f.workspacePath, destBasePath)
// requirement is new, so we should move the files and add it to the mod file
if existing == nil {
if err := move(req, tmpPath, destPath); err != nil {
return isNew, err
}
f.require = append(f.require, req)
isNew = true
return isNew, nil
}
if upgrade {
latestTag, err := r.latestTag()
if err != nil {
return isNew, err
}
if latestTag == "" {
return isNew, fmt.Errorf("repo does not have a tag")
}
req.version = latestTag
}
c, err := compareVersions(existing.version, req.version)
if err != nil {
return isNew, err
}
// the existing requirement is newer or equal so we skip installation
if c >= 0 {
return isNew, nil
}
// the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing
// requirement and replace the code in the /pkg folder
existing.version = req.version
if err = r.checkout(req.version); err != nil {
return isNew, err
}
if err = replace(req, tmpPath, destPath); err != nil {
return isNew, err
}
isNew = true
return isNew, nil
}
func (f *file) write() error {
return ioutil.WriteFile(path.Join(f.workspacePath, filePath), f.contents().Bytes(), 0600)
}
func (f *file) contents() *bytes.Buffer {
var b bytes.Buffer
for _, r := range f.require {
b.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.version))
}
return &b
}
func (f *file) search(r *require) *require {
for _, existing := range f.require {
if existing.fullPath() == r.fullPath() {
return existing
}
}
return nil
}
type require struct {
repo string
path string
version string
cloneRepo string
clonePath string
}
func (r *require) fullPath() string {
return path.Join(r.repo, r.path)
}
func move(r *require, sourceRepoPath, destBasePath string) error {
destPath := path.Join(destBasePath, r.fullPath())
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
return err
}
if err := os.Rename(path.Join(sourceRepoPath, r.path), destPath); err != nil {
return err
}
return nil
}
func replace(r *require, sourceRepoPath, destBasePath string) error {
if err := os.RemoveAll(path.Join(destBasePath, r.fullPath())); err != nil {
return err
}
return move(r, sourceRepoPath, destBasePath)
}

View File

@ -0,0 +1,49 @@
package mod
import (
"strings"
"testing"
)
func TestReadFile(t *testing.T) {
cases := []struct {
name string
input string
want *file
}{
{
name: "module file with valid dependencies",
input: `
github.com/tjovicic/test xyz
github.com/bla/bla abc
`,
want: &file{
require: []*require{
{
repo: "github.com/tjovicic/test",
path: "",
version: "xyz",
},
{
repo: "github.com/bla/bla",
path: "",
version: "abc",
},
},
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := read(strings.NewReader(c.input))
if err != nil {
t.Error(err)
}
if len(got.require) != len(c.want.require) {
t.Errorf("requires length differs: want %d, got %d", len(c.want.require), len(got.require))
}
})
}
}

110
cmd/dagger/cmd/mod/get.go Normal file
View File

@ -0,0 +1,110 @@
package mod
import (
"github.com/hashicorp/go-version"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"go.dagger.io/dagger/cmd/dagger/cmd/common"
"go.dagger.io/dagger/cmd/dagger/logger"
"go.dagger.io/dagger/telemetry"
)
var getCmd = &cobra.Command{
Use: "get [packages]",
Short: "download and install dependencies",
Args: cobra.MaximumNArgs(1),
PreRun: func(cmd *cobra.Command, args []string) {
// Fix Viper bug for duplicate flags:
// https://github.com/spf13/viper/issues/233
if err := viper.BindPFlags(cmd.Flags()); err != nil {
panic(err)
}
},
Run: func(cmd *cobra.Command, args []string) {
lg := logger.New()
ctx := lg.WithContext(cmd.Context())
workspace := common.CurrentWorkspace(ctx)
st := common.CurrentEnvironmentState(ctx, workspace)
doneCh := common.TrackWorkspaceCommand(ctx, cmd, workspace, st, &telemetry.Property{
Name: "packages",
Value: args,
})
// read mod file in the current dir
modFile, err := readPath(workspace.Path)
if err != nil {
lg.Fatal().Err(err).Msg("error loading module file")
}
// parse packages to install
var packages []*require
var upgrade bool
if len(args) == 0 {
lg.Info().Msg("upgrading installed packages...")
packages = modFile.require
upgrade = true
} else {
for _, arg := range args {
p, err := parseArgument(arg)
if err != nil {
lg.Error().Err(err).Msgf("error parsing package %s", arg)
continue
}
packages = append(packages, p)
}
}
// download packages
for _, p := range packages {
isNew, err := modFile.processRequire(p, upgrade)
if err != nil {
lg.Error().Err(err).Msgf("error processing package %s", p.repo)
}
if isNew {
lg.Info().Msgf("downloading %s:%v", p.repo, p.version)
}
}
// write to mod file in the current dir
if err = modFile.write(); err != nil {
lg.Error().Err(err).Msg("error writing to mod file")
}
<-doneCh
},
}
func compareVersions(reqV1, reqV2 string) (int, error) {
v1, err := version.NewVersion(reqV1)
if err != nil {
return 0, err
}
v2, err := version.NewVersion(reqV2)
if err != nil {
return 0, err
}
if v1.LessThan(v2) {
return -1, nil
}
if v1.Equal(v2) {
return 0, nil
}
return 1, nil
}
func init() {
getCmd.Flags().String("private-key-file", "", "Private ssh key")
getCmd.Flags().String("private-key-password", "", "Private ssh key password")
if err := viper.BindPFlags(getCmd.Flags()); err != nil {
panic(err)
}
}

128
cmd/dagger/cmd/mod/repo.go Normal file
View File

@ -0,0 +1,128 @@
package mod
import (
"fmt"
"os"
"path"
"sort"
"strings"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/hashicorp/go-version"
)
type repo struct {
localPath string
contents *git.Repository
}
func clone(require *require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) {
o := git.CloneOptions{
URL: fmt.Sprintf("https://%s", require.cloneRepo),
}
if privateKeyFile != "" {
publicKeys, err := ssh.NewPublicKeysFromFile("git", privateKeyFile, privateKeyPassword)
if err != nil {
return nil, err
}
o.Auth = publicKeys
o.URL = fmt.Sprintf("git@%s", strings.Replace(require.cloneRepo, "/", ":", 1))
}
r, err := git.PlainClone(dir, false, &o)
if err != nil {
return nil, err
}
rr := &repo{
localPath: dir,
contents: r,
}
if require.version == "" {
latestTag, err := rr.latestTag()
if err != nil {
return nil, err
}
if latestTag == "" {
return nil, fmt.Errorf("no git tags found in the repo")
}
require.version = latestTag
}
if err := rr.checkout(require.version); err != nil {
return nil, err
}
if _, err := os.Stat(path.Join(dir, require.clonePath, filePath)); err != nil {
return nil, fmt.Errorf("repo does not contain %s", filePath)
}
return rr, nil
}
func (r *repo) checkout(version string) error {
h, err := r.contents.ResolveRevision(plumbing.Revision(version))
if err != nil {
return err
}
w, err := r.contents.Worktree()
if err != nil {
return err
}
err = w.Checkout(&git.CheckoutOptions{
Hash: *h,
})
if err != nil {
return err
}
return nil
}
func (r *repo) listTags() ([]string, error) {
iter, err := r.contents.Tags()
if err != nil {
return nil, err
}
var tags []string
if err := iter.ForEach(func(ref *plumbing.Reference) error {
tags = append(tags, ref.Name().Short())
return nil
}); err != nil {
return nil, err
}
return tags, nil
}
func (r *repo) latestTag() (string, error) {
versionsRaw, err := r.listTags()
if err != nil {
return "", err
}
versions := make([]*version.Version, len(versionsRaw))
for i, raw := range versionsRaw {
v, _ := version.NewVersion(raw)
versions[i] = v
}
sort.Sort(version.Collection(versions))
if len(versions) == 0 {
return "", nil
}
return versions[len(versions)-1].Original(), nil
}

View File

@ -0,0 +1,93 @@
package mod
import (
"io/ioutil"
"os"
"testing"
)
func TestClone(t *testing.T) {
cases := []struct {
name string
require require
privateKeyFile string
privateKeyPassword string
}{
{
name: "resolving shorter hash version",
require: require{
cloneRepo: "github.com/tjovicic/dagger-modules",
clonePath: "gcpcloudrun",
version: "26a1d46d1b3c",
},
},
{
name: "resolving branch name",
require: require{
cloneRepo: "github.com/tjovicic/dagger-modules",
clonePath: "gcpcloudrun",
version: "main",
},
},
{
name: "resolving tag",
require: require{
cloneRepo: "github.com/tjovicic/dagger-modules",
clonePath: "gcpcloudrun",
version: "v0.3",
},
},
{
name: "Dagger private test repo",
require: require{
cloneRepo: "github.com/dagger/test",
clonePath: "",
version: "main",
},
privateKeyFile: "./test-ssh-keys/id_ed25519_test",
privateKeyPassword: "",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "clone")
if err != nil {
t.Fatal("error creating tmp dir")
}
defer os.Remove(tmpDir)
_, err = clone(&c.require, tmpDir, c.privateKeyFile, c.privateKeyPassword)
if err != nil {
t.Error(err)
}
})
}
}
func TestListTags(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "clone")
if err != nil {
t.Fatal("error creating tmp dir")
}
defer os.Remove(tmpDir)
r, err := clone(&require{
cloneRepo: "github.com/tjovicic/dagger-modules",
clonePath: "gcpcloudrun",
version: "",
}, tmpDir, "", "")
if err != nil {
t.Fatal(err)
}
tags, err := r.listTags()
if err != nil {
t.Error(err)
}
if len(tags) == 0 {
t.Errorf("could not list repo tags")
}
}

View File

@ -0,0 +1,15 @@
package mod
import "github.com/spf13/cobra"
// Cmd exposes the top-level command
var Cmd = &cobra.Command{
Use: "mod",
Short: "Manage an environment's dependencies",
}
func init() {
Cmd.AddCommand(
getCmd,
)
}

View File

@ -0,0 +1,7 @@
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
QyNTUxOQAAACCpGsk8WLx7gXCXX1muGhKjlkqaqykF1X198WQMkBO2pwAAAKC5Ec8WuRHP
FgAAAAtzc2gtZWQyNTUxOQAAACCpGsk8WLx7gXCXX1muGhKjlkqaqykF1X198WQMkBO2pw
AAAEBXE9Uht+QHuyK7+yYcZFVWOJ3qkhUh/wn289nDKDPHKakayTxYvHuBcJdfWa4aEqOW
SpqrKQXVfX3xZAyQE7anAAAAGnRpaG9taXIuam92aWNpY0B0b3B0YWwuY29tAQID
-----END OPENSSH PRIVATE KEY-----

View File

@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKkayTxYvHuBcJdfWa4aEqOWSpqrKQXVfX3xZAyQE7an tihomir.jovicic@toptal.com

View File

@ -8,6 +8,7 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
"go.dagger.io/dagger/cmd/dagger/cmd/input"
"go.dagger.io/dagger/cmd/dagger/cmd/mod"
"go.dagger.io/dagger/cmd/dagger/cmd/output"
"go.dagger.io/dagger/cmd/dagger/logger"
"go.dagger.io/dagger/keychain"
@ -66,6 +67,7 @@ func init() {
output.Cmd,
versionCmd,
docCmd,
mod.Cmd,
)
if err := viper.BindPFlags(rootCmd.PersistentFlags()); err != nil {

View File

@ -0,0 +1,118 @@
---
slug: /1011/package-manager/
---
# Manage packages using the package manager
This tutorial illustrates how to install and upgrade packages using Dagger package manager.
## Installing a package
### Initializing workspace
Create an empty directory for your new Dagger workspace:
```shell
mkdir workspace
cd workspace
```
As described in the previous tutorials, initialize your Dagger workspace:
```shell
dagger init
dagger new test
```
That will create 2 directories: `.dagger` and `cue.mod` where our package will reside:
```shell
.
├── cue.mod
│ ├── module.cue
│ ├── pkg
│ └── usr
├── .dagger
│ └── env
│ └── test
```
### Install
In our example we will use `gcpcloudrun` module from [github](https://github.com/tjovicic/dagger-modules/blob/main/gcpcloudrun/source.cue)
Let's first add it to our `source.cue` file:
```cue title="./source.cue"
package main
import (
"github.com/tjovicic/dagger-modules/gcpcloudrun"
)
run: gcpcloudrun.#Run
```
To install it just run
```shell
dagger mod get github.com/tjovicic/dagger-modules/gcpcloudrun@v0.1
```
It should pull the `v0.1` version from Github, leave a copy in `cue.mod/pkg` and reflect the change in
`cue.mod/dagger.mod.cue` file:
```shell
cue.mod/pkg/github.com/
└── tjovicic
└── dagger-modules
└── gcpcloudrun
├── cue.mod
├── README.md
└── source.cue
```
```cue title="./cue.mod/dagger.mod"
github.com/tjovicic/dagger-modules/gcpcloudrun v0.1
```
Querying the current setup with `dagger query` should return a valid result:
```json
{
"run": {
"creds": {
"username": "oauth2accesstoken"
},
"deploy": {
"platform": "managed",
"port": "80"
},
"push": {
"auth": {
"username": "oauth2accesstoken"
},
"push": {}
}
}
}
```
### Upgrading
Now that you've successfully installed a package, let's try to upgrade it.
```shell
dagger mod get github.com/tjovicic/dagger-modules/gcpcloudrun@v0.2
```
You should see similar output:
```shell
12:25PM INF system | downloading github.com/tjovicic/dagger-modules:v0.2
```
And `cue.mod/dagger.mod.cue` should reflect the new version:
```cue title="./cue.mod/dagger.mod"
github.com/tjovicic/dagger-modules/gcpcloudrun v0.2
```

View File

@ -29,22 +29,24 @@ const (
func init() {
bi, ok := debug.ReadBuildInfo()
if !ok {
panic("unable to retrieve build info")
return
}
for _, d := range bi.Deps {
if d.Path == "github.com/moby/buildkit" {
vendoredVersion = d.Version
break
}
}
if vendoredVersion == "" {
panic("failed to solve vendored buildkit version")
}
}
func Start(ctx context.Context) (string, error) {
lg := log.Ctx(ctx)
if vendoredVersion == "" {
return "", fmt.Errorf("vendored version is empty")
}
// Attempt to detect the current buildkit version
currentVersion, err := getBuildkitVersion(ctx)
if err != nil {
@ -66,7 +68,7 @@ func Start(ctx context.Context) (string, error) {
Info().
Str("version", vendoredVersion).
Msg("upgrading buildkit")
if err := remvoveBuildkit(ctx); err != nil {
if err := removeBuildkit(ctx); err != nil {
return "", err
}
} else {
@ -183,7 +185,7 @@ func waitBuildkit(ctx context.Context) error {
return errors.New("buildkit failed to respond")
}
func remvoveBuildkit(ctx context.Context) error {
func removeBuildkit(ctx context.Context) error {
lg := log.
Ctx(ctx)