Merge pull request #149 from dagger/autostart-buildkit

automatically start/upgrade buildkitd

.github/workflows/ci.yml

@@ -17,16 +17,21 @@ jobs:
           go-version: 1.16
         id: go
 
+      - name: Check out
+        uses: actions/checkout@v2
+
       - name: Install Dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends shellcheck
+
+          export CUE_VERSION="$(grep cue ./go.mod | cut -d' ' -f2)"
+          export CUE_TARBALL="cue_$(echo "$CUE_VERSION" | sed 's/^v//')_Linux_x86_64.tar.gz"
+
+          echo "Installing cue version $CUE_VERSION"
           curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sudo sh -s -- -b /usr/local/bin v1.23.8
-          curl -L https://github.com/cuelang/cue/releases/download/v0.3.0-beta.4/cue_0.3.0-beta.4_Linux_x86_64.tar.gz | sudo tar zxf - -C /usr/local/bin
-
-      - name: Check out
-        uses: actions/checkout@v2
 
+          curl -L https://github.com/cuelang/cue/releases/download/${CUE_VERSION}/${CUE_TARBALL} | sudo tar zxf - -C /usr/local/bin
       - name: Build
         run: |
           make
@@ -35,10 +40,6 @@ jobs:
         run: |
           make lint
 
-      - name: Start buildkit
-        run: |
-          docker run -d --name buildkitd --privileged moby/buildkit:v0.8.2
-
       - name: Integration test
         run: |
           make integration

Makefile

@@ -19,13 +19,21 @@ cuefmt:
 	@(cue fmt -s ./examples/*)
 
 .PHONY: lint
-lint: cuefmt
+lint: cuefmt check-buildkit-version
 	golangci-lint run
 	@test -z "$$(git status -s . | grep -e "^ M"  | grep .cue | cut -d ' ' -f3 | tee /dev/stderr)"
 
+.PHONY: check-buildkit-version
+check-buildkit-version:
+	@test \
+		"$(shell grep buildkit ./go.mod | cut -d' ' -f2)" = \
+		"$(shell grep ' = "v' ./pkg/buildkitd/buildkitd.go | sed -E 's/^.*version.*=.*\"(v.*)\"/\1/' )" \
+		|| { echo buildkit version mismatch go.mod != pkg/buildkitd/buildkitd.go ; exit 1; }
+
 .PHONY: integration
 integration: dagger-debug
 	# Self-diagnostics
 	./tests/test-test.sh 2>/dev/null
 	# Actual integration tests
 	DAGGER_BINARY="./cmd/dagger/dagger-debug" time ./tests/test.sh all
+

README.md

@@ -74,15 +74,7 @@ $ make
 $ cp ./cmd/dagger/dagger /usr/local/bin
 ```
 
-3. Run [buildkitd](https://github.com/moby/buildkit) on your local machine. The simplest way to do this is using [Docker](https://docker.com): `docker run -d --name buildkitd --privileged moby/buildkit:latest`
-
-On a machine with Docker installed, run:
-
-```
-$ docker run -d --name buildkitd --privileged moby/buildkit:latest
-```
-
-4. Compute a test configuration
+3. Compute a test configuration
 
 Currently `dagger` can only do one thing: compute a configuration with optional inputs, and print the result.
 

dagger/client.go

@@ -22,15 +22,12 @@ import (
 	bkgw "github.com/moby/buildkit/frontend/gateway/client"
 
 	// docker output
+	"dagger.io/go/pkg/buildkitd"
 	"dagger.io/go/pkg/progressui"
 
 	"dagger.io/go/dagger/compiler"
 )
 
-const (
-	defaultBuildkitHost = "docker-container://buildkitd"
-)
-
 // A dagger client
 type Client struct {
 	c *bk.Client
@@ -41,7 +38,12 @@ func NewClient(ctx context.Context, host string) (*Client, error) {
 		host = os.Getenv("BUILDKIT_HOST")
 	}
 	if host == "" {
-		host = defaultBuildkitHost
+		h, err := buildkitd.Start(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		host = h
 	}
 	c, err := bk.New(ctx, host)
 	if err != nil {

pkg/buildkitd/buildkitd.go

@@ -0,0 +1,170 @@
+package buildkitd
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/docker/distribution/reference"
+	"github.com/rs/zerolog/log"
+)
+
+const (
+	image         = "moby/buildkit"
+	version       = "v0.8.2"
+	imageVersion  = image + ":" + version
+	containerName = "dagger-buildkitd"
+	volumeName    = "dagger-buildkitd"
+)
+
+func Start(ctx context.Context) (string, error) {
+	lg := log.Ctx(ctx)
+
+	// Attempt to detect the current buildkit version
+	currentVersion, err := getBuildkitVersion(ctx)
+	if err != nil {
+		// If that failed, it might either be because buildkitd is not running
+		// or because the docker CLI is out of service.
+		if err := checkDocker(ctx); err != nil {
+			return "", err
+		}
+
+		currentVersion = ""
+		lg.Debug().Msg("no buildkit daemon detected")
+	} else {
+		lg.Debug().Str("version", currentVersion).Msg("detected buildkit version")
+	}
+
+	if currentVersion != version {
+		if currentVersion != "" {
+			lg.
+				Info().
+				Str("version", version).
+				Msg("upgrading buildkit")
+			if err := remvoveBuildkit(ctx); err != nil {
+				return "", err
+			}
+		} else {
+			lg.
+				Info().
+				Str("version", version).
+				Msg("starting buildkit")
+		}
+		if err := startBuildkit(ctx); err != nil {
+			return "", err
+		}
+	}
+
+	return fmt.Sprintf("docker-container://%s", containerName), nil
+}
+
+// ensure the docker CLI is available and properly set up (e.g. permissions to
+// communicate with the daemon, etc)
+func checkDocker(ctx context.Context) error {
+	cmd := exec.CommandContext(ctx, "docker", "info")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		log.
+			Ctx(ctx).
+			Error().
+			Err(err).
+			Bytes("output", output).
+			Msg("failed to run docker")
+		return err
+	}
+
+	return nil
+}
+
+func startBuildkit(ctx context.Context) error {
+	lg := log.
+		Ctx(ctx).
+		With().
+		Str("version", version).
+		Logger()
+
+	lg.Debug().Msg("pulling buildkit image")
+	cmd := exec.CommandContext(ctx,
+		"docker",
+		"pull",
+		imageVersion,
+	)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		lg.
+			Error().
+			Err(err).
+			Bytes("output", output).
+			Msg("failed to pull buildkit image")
+		return err
+	}
+
+	cmd = exec.CommandContext(ctx,
+		"docker",
+		"run",
+		"-d",
+		"--restart", "always",
+		"-v", volumeName+":/var/lib/buildkit",
+		"--name", containerName,
+		"--privileged",
+		imageVersion,
+	)
+	output, err = cmd.CombinedOutput()
+	if err != nil {
+		log.
+			Ctx(ctx).
+			Error().
+			Err(err).
+			Bytes("output", output).
+			Msg("unable to start buildkitd")
+		return err
+	}
+	return nil
+}
+
+func remvoveBuildkit(ctx context.Context) error {
+	lg := log.
+		Ctx(ctx)
+
+	cmd := exec.CommandContext(ctx,
+		"docker",
+		"rm",
+		"-fv",
+		containerName,
+	)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		lg.
+			Error().
+			Err(err).
+			Bytes("output", output).
+			Msg("failed to stop buildkit")
+		return err
+	}
+
+	return nil
+}
+
+func getBuildkitVersion(ctx context.Context) (string, error) {
+	cmd := exec.CommandContext(ctx,
+		"docker",
+		"inspect",
+		"--format",
+		"{{.Config.Image}}",
+		containerName,
+	)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", err
+	}
+	ref, err := reference.ParseNormalizedNamed(strings.TrimSpace(string(output)))
+	if err != nil {
+		return "", err
+	}
+	tag, ok := ref.(reference.Tagged)
+	if !ok {
+		return "", fmt.Errorf("failed to parse image: %s", output)
+	}
+	return tag.Tag(), nil
+}