Merge pull request #1984 from aluzzardi/ci-improvement

CI dogfood improvements

.github/workflows/dagger-ci.yml

@@ -2,42 +2,84 @@ name: "Dagger CI"
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
     paths:
-      - '**.sh'
+      - "**.sh"
-      - '**.bash'
+      - "**.bash"
-      - '**.go'
+      - "**.go"
-      - '**.cue'
+      - "**.cue"
-      - '**.bats'
+      - "**.bats"
-      - 'Makefile'
+      - "Makefile"
-      - 'go.mod'
+      - "go.mod"
-      - 'go.sum'
+      - "go.sum"
-      - '.github/workflows/dagger-ci.yml'
+      - ".github/workflows/dagger-ci.yml"
   pull_request:
-    branches: [ main ]
+    branches: [main]
     paths:
-      - '**.sh'
+      - "**.sh"
-      - '**.bash'
+      - "**.bash"
-      - '**.go'
+      - "**.go"
-      - '**.cue'
+      - "**.cue"
-      - '**.bats'
+      - "**.bats"
-      - 'Makefile'
+      - "Makefile"
-      - 'go.mod'
+      - "go.mod"
-      - 'go.sum'
+      - "go.sum"
-      - '.github/workflows/dagger-ci.yml'
+      - ".github/workflows/dagger-ci.yml"
 
 env:
   DAGGER_LOG_FORMAT: plain
 
 jobs:
   build:
+    name: "Build"
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Dagger CI
+      - name: "Setup Go"
-        uses: dagger/dagger-for-github@v2
+        uses: actions/setup-go@v1
         with:
-          workdir: ci
+          go-version: 1.16
-          args: do build
+
+      - name: "Expose GitHub Runtime"
+        uses: crazy-max/ghaction-github-runtime@v1
+
+      - name: Prepare dagger
+        run: |
+          make dagger
+          cp ./cmd/dagger/dagger /usr/local/bin
+
+      - name: Build
+        env:
+          DAGGER_CACHE_TO: "type=gha,mode=max,scope=dagger-ci-build"
+          DAGGER_CACHE_FROM: "type=gha,scope=dagger-ci-build"
+        run: |
+          dagger do build
+
+  lint:
+    name: "Lint"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.16
+
+      - name: "Expose GitHub Runtime"
+        uses: crazy-max/ghaction-github-runtime@v1
+
+      - name: Prepare dagger
+        run: |
+          make dagger
+          cp ./cmd/dagger/dagger /usr/local/bin
+
+      - name: Lint
+        env:
+          DAGGER_CACHE_TO: "type=gha,mode=max,scope=dagger-ci-lint"
+          DAGGER_CACHE_FROM: "type=gha,scope=dagger-ci-lint"
+        run: |
+          dagger do lint

ci.cue

@@ -0,0 +1,150 @@
+package main
+
+import (
+	"dagger.io/dagger"
+
+	"universe.dagger.io/bash"
+	"universe.dagger.io/alpine"
+	"universe.dagger.io/docker"
+	"universe.dagger.io/go"
+
+	"github.com/dagger/dagger/ci/golangci"
+	"github.com/dagger/dagger/ci/shellcheck"
+	"github.com/dagger/dagger/ci/markdownlint"
+)
+
+dagger.#Plan & {
+	// FIXME: Ideally we would want to automatically set the platform's arch identical to the host
+	// to avoid the performance hit caused by qemu (linter goes from <3s to >3m when arch is x86)
+	// Uncomment if running locally on Mac M1 to bypass qemu
+	// platform: "linux/aarch64"
+	platform: "linux/amd64"
+
+	client: filesystem: ".": read: exclude: [
+		"bin",
+		"**/node_modules",
+		"cmd/dagger/dagger",
+		"cmd/dagger/dagger-debug",
+	]
+	client: filesystem: "./bin": write: contents: actions.build.output
+
+	actions: {
+		_source: client.filesystem["."].read.contents
+
+		// FIXME: this can be removed once `go` supports built-in VCS info
+		version: {
+			_image: alpine.#Build & {
+				packages: bash: _
+				packages: curl: _
+				packages: git:  _
+			}
+
+			_revision: bash.#Run & {
+				input:   _image.output
+				workdir: "/src"
+				mounts: source: {
+					dest:     "/src"
+					contents: _source
+				}
+
+				script: contents: #"""
+					printf "$(git rev-parse --short HEAD)" > /revision
+					"""#
+				export: files: "/revision": string
+			}
+
+			output: _revision.export.files["/revision"]
+		}
+
+		build: go.#Build & {
+			source:  _source
+			package: "./cmd/dagger/"
+			os:      client.platform.os
+			arch:    client.platform.arch
+
+			ldflags: "-s -w -X go.dagger.io/dagger/version.Revision=\(version.output)"
+
+			env: {
+				CGO_ENABLED: "0"
+				// Makes sure the linter and unit tests complete before starting the build
+				// "__depends_lint":  "\(goLint.exit)"
+				// "__depends_tests": "\(goTest.exit)"
+			}
+		}
+
+		// Go unit tests
+		test: go.#Test & {
+			// container: image: _goImage.output
+			source:  _source
+			package: "./..."
+
+			// FIXME: doesn't work with CGO_ENABLED=0
+			// command: flags: "-race": true
+
+			env: {
+				// FIXME: removing this complains about lack of gcc
+				CGO_ENABLED: "0"
+			}
+		}
+
+		lint: {
+			go: golangci.#Lint & {
+				source:  _source
+				version: "1.45"
+			}
+
+			shell: shellcheck.#Lint & {
+				source: _source
+			}
+
+			markdown: markdownlint.#Lint & {
+				source: _source
+				files: ["./docs", "README.md"]
+			}
+
+			cue: docker.#Build & {
+				// FIXME: spin off into its own package?
+				steps: [
+					alpine.#Build & {
+						packages: bash: _
+						packages: curl: _
+						packages: git:  _
+					},
+
+					docker.#Copy & {
+						contents: _source
+						source:   "go.mod"
+						dest:     "go.mod"
+					},
+
+					// Install CUE
+					bash.#Run & {
+						script: contents: #"""
+								export CUE_VERSION="$(grep cue ./go.mod | cut -d' ' -f2 | head -1 | sed -E 's/\.[[:digit:]]\.[[:alnum:]]+-[[:alnum:]]+$//')"
+								export CUE_TARBALL="cue_${CUE_VERSION}_linux_amd64.tar.gz"
+								echo "Installing cue version $CUE_VERSION"
+								curl -L "https://github.com/cue-lang/cue/releases/download/${CUE_VERSION}/${CUE_TARBALL}" | tar zxf - -C /usr/local/bin
+								cue version
+						"""#
+					},
+
+					// CACHE: copy only *.cue files
+					docker.#Copy & {
+						contents: _source
+						include: ["*.cue"]
+						dest: "/cue"
+					},
+
+					// LINT
+					bash.#Run & {
+						workdir: "/cue"
+						script: contents: #"""
+							find . -name '*.cue' -not -path '*/cue.mod/*' -print | time xargs -t -n 1 -P 8 cue fmt -s
+							test -z "$(git status -s . | grep -e "^ M"  | grep "\.cue" | cut -d ' ' -f3 | tee /dev/stderr)"
+							"""#
+					},
+				]
+			}
+		}
+	}
+}

ci/.gitignore

@@ -1 +0,0 @@
-build/*

ci/README.md

@@ -1,7 +0,0 @@
-# Dagger CI
-
-WORK IN PROGRESS
-
-This is the home of the new CI
-
-Reference: https://github.com/dagger/dagger/issues/1549

ci/golangci/lint.cue

@@ -0,0 +1,37 @@
+package golangci
+
+import (
+	"dagger.io/dagger"
+
+	"universe.dagger.io/docker"
+	"universe.dagger.io/go"
+)
+
+// Lint using golangci-lint
+#Lint: {
+	// Source code
+	source: dagger.#FS
+
+	// golangci-lint version
+	version: *"1.45" | string
+
+	// timeout
+	timeout: *"5m" | string
+
+	_image: docker.#Pull & {
+		source: "golangci/golangci-lint:v\(version)"
+	}
+
+	container: go.#Container & {
+		"source": source
+		input:    _image.output
+		command: {
+			name: "golangci-lint"
+			flags: {
+				run:         true
+				"-v":        true
+				"--timeout": timeout
+			}
+		}
+	}
+}

ci/images.cue

@@ -1,80 +0,0 @@
-package main
-
-import (
-	"universe.dagger.io/docker"
-)
-
-let GoVersion = "1.17"
-let GolangCILintVersion = "1.44.0"
-let CUEVersion = "0.4.2"
-
-// Base container images used for the CI
-#Images: {
-
-	// base image to build go binaries
-	goBuilder:  _goBuilder.output
-	_goBuilder: docker.#Build & {
-		_packages: ["bash", "git", "alpine-sdk"]
-
-		steps: [
-			docker.#Pull & {
-				source: "index.docker.io/golang:\(GoVersion)-alpine"
-			},
-			for pkg in _packages {
-				docker.#Run & {
-					command: {
-						name: "apk"
-						args: ["add", pkg]
-						flags: {
-							"-U":         true
-							"--no-cache": true
-						}
-					}
-				}
-			},
-		]
-	}
-
-	// base image for the Go linter
-	// https://golangci-lint.run/usage/install/#docker
-	goLinter:  _goLinter.output
-	_goLinter: docker.#Pull & {
-		source: "index.docker.io/golangci/golangci-lint:v\(GolangCILintVersion)"
-	}
-
-	// base image for CUE cli + alpine distrib
-	cue: _cue._alpine.output
-	_cue: {
-		_cueBinary: docker.#Pull & {
-			source: "index.docker.io/cuelang/cue:\(CUEVersion)"
-		}
-
-		_alpine: docker.#Build & {
-			_packages: ["bash", "git"]
-
-			steps: [
-				docker.#Pull & {
-					source: "index.docker.io/alpine:3"
-				},
-				for pkg in _packages {
-					docker.#Run & {
-						command: {
-							name: "apk"
-							args: ["add", pkg]
-							flags: {
-								"-U":         true
-								"--no-cache": true
-							}
-						}
-					}
-				},
-				docker.#Copy & {
-					// input:    _alpine.output
-					contents: _cueBinary.output.rootfs
-					source:   "/usr/bin/cue"
-					dest:     "/usr/bin/cue"
-				},
-			]
-		}
-	}
-}

ci/main.cue

@@ -1,99 +0,0 @@
-package main
-
-import (
-	"dagger.io/dagger"
-	"dagger.io/dagger/core"
-	"universe.dagger.io/bash"
-)
-
-dagger.#Plan & {
-
-	// FIXME: Ideally we would want to automatically set the platform's arch identical to the host
-	// to avoid the performance hit caused by qemu (linter goes from <3s to >3m when arch is x86)
-	// Uncomment if running locally on Mac M1 to bypass qemu
-	// platform: "linux/aarch64"
-	platform: "linux/amd64"
-
-	client: filesystem: "./build": write: contents: actions.build.export.directories["/build"]
-
-	actions: {
-		_mountGoCache: {
-			mounts: "go mod cache": {
-				dest:     "/root/.gocache"
-				contents: core.#CacheDir & {
-					id: "go mod cache"
-				}
-			}
-			env: GOMODCACHE: mounts["go mod cache"].dest
-		}
-
-		_mountSourceCode: {
-			mounts: "dagger source code": {
-				contents: _source.output
-				dest:     "/usr/src/dagger"
-			}
-			workdir: mounts["dagger source code"].dest
-		}
-
-		_baseImages: #Images
-
-		// Go build the dagger binary
-		// depends on goLint and goTest to complete successfully
-		build: bash.#Run & {
-			_mountSourceCode
-			_mountGoCache
-
-			input: _baseImages.goBuilder
-
-			env: {
-				GOOS:        client.platform.os
-				GOARCH:      client.platform.arch
-				CGO_ENABLED: "0"
-				// Makes sure the linter and unit tests complete before starting the build
-				"__depends_lint":  "\(goLint.exit)"
-				"__depends_tests": "\(goTest.exit)"
-			}
-
-			script: contents: #"""
-				mkdir -p /build
-				git_revision=$(git rev-parse --short HEAD)
-				go build -v -o /build/dagger \
-				 -ldflags '-s -w -X go.dagger.io/dagger/version.Revision='${git_revision} \
-				 ./cmd/dagger/
-				"""#
-
-			export: directories: "/build": _
-		}
-
-		// Go unit tests
-		goTest: bash.#Run & {
-			_mountSourceCode
-			_mountGoCache
-
-			input: _baseImages.goBuilder
-			script: contents: "go test -race -v ./..."
-		}
-
-		// Go lint using golangci-lint
-		goLint: bash.#Run & {
-			_mountSourceCode
-			_mountGoCache
-
-			input: _baseImages.goLinter
-			script: contents: "golangci-lint run -v --timeout 5m"
-		}
-
-		// CUE lint
-		cueLint: bash.#Run & {
-			_mountSourceCode
-
-			input: _baseImages.cue
-			script: contents: #"""
-				# Format the cue code
-				find . -name '*.cue' -not -path '*/cue.mod/*' -print | time xargs -n 1 -P 8 cue fmt -s
-				# Check that all formatted files where committed
-				test -z $(git status -s . | grep -e '^ M'  | grep .cue | cut -d ' ' -f3)
-				"""#
-		}
-	}
-}

ci/markdownlint/markdownlint.cue

@@ -0,0 +1,36 @@
+package markdownlint
+
+import (
+	"dagger.io/dagger"
+
+	"universe.dagger.io/docker"
+)
+
+#Lint: {
+	// Source code
+	source: dagger.#FS
+
+	// shellcheck version
+	version: *"0.31.1" | string
+
+	// Files to lint
+	files: [...string]
+
+	_image: docker.#Pull & {
+		source: "tmknom/markdownlint:\(version)"
+	}
+
+	container: docker.#Run & {
+		input: _image.output
+		mounts: "source": {
+			dest:     "/src"
+			contents: source
+		}
+		workdir: "/src"
+		command: {
+			// FIXME: this should not be required
+			name: "markdownlint"
+			args: files
+		}
+	}
+}

ci/shellcheck/shellcheck.cue

@@ -0,0 +1,34 @@
+package shellcheck
+
+import (
+	"dagger.io/dagger"
+
+	"universe.dagger.io/docker"
+)
+
+#Lint: {
+	// Source code
+	source: dagger.#FS
+
+	// shellcheck version
+	version: *"0.8.0" | string
+
+	_image: docker.#Pull & {
+		source: "koalaman/shellcheck-alpine:v\(version)"
+	}
+
+	container: docker.#Run & {
+		input: _image.output
+		mounts: "source": {
+			dest:     "/src"
+			contents: source
+		}
+		workdir: "/src"
+		command: {
+			name: "sh"
+			args: ["-c", #"""
+				shellcheck $(find . -type f \( -iname \*.bats -o -iname \*.bash -o -iname \*.sh \) -not -path "*/node_modules/*" -not -path "*/bats-*/*")
+				"""#]
+		}
+	}
+}

cue.mod/module.cue

@@ -1 +1 @@
-module: ""
+module: "github.com/dagger/dagger"

pkg/universe.dagger.io/go/build.cue

@@ -13,10 +13,10 @@ import (
 	package: *"." | string
 
 	// Target architecture
-	arch: *"amd64" | string
+	arch?: string
 
 	// Target OS
-	os: *"linux" | string
+	os?: string
 
 	// Build tags to use for building
 	tags: *"" | string
@@ -30,10 +30,15 @@ import (
 		"source": source
 		"env": {
 			env
-			GOOS:   os
+			if os != _|_ {
-			GOARCH: arch
+				GOOS: os
+			}
+			if arch != _|_ {
+				GOARCH: arch
+			}
 		}
 		command: {
+			name: "go"
 			args: [package]
 			flags: {
 				build:      true

pkg/universe.dagger.io/go/container.cue

@@ -18,25 +18,31 @@ import (
 	// Use go image
 	_image: #Image
 
-	_sourcePath: "/src"
+	_sourcePath:     "/src"
-	_cachePath:  "/root/.cache/gocache"
+	_modCachePath:   "/root/.cache/go-mod"
+	_buildCachePath: "/root/.cache/go-build"
 
 	docker.#Run & {
 		input:   *_image.output | docker.#Image
-		workdir: "/src"
+		workdir: _sourcePath
-		command: name: "go"
 		mounts: {
 			"source": {
 				dest:     _sourcePath
 				contents: source
 			}
-			"go assets cache": {
+			"go mod cache": {
 				contents: core.#CacheDir & {
-					id: "\(name)_assets"
+					id: "\(name)_mod"
 				}
-				dest: _cachePath
+				dest: _modCachePath
+			}
+			"go build cache": {
+				contents: core.#CacheDir & {
+					id: "\(name)_build"
+				}
+				dest: _buildCachePath
 			}
 		}
-		env: GOMODCACHE: _cachePath
+		env: GOMODCACHE: _modCachePath
 	}
 }

pkg/universe.dagger.io/go/test.cue

@@ -7,6 +7,7 @@ package go
 
 	#Container & {
 		command: {
+			name: "go"
 			args: [package]
 			flags: {
 				test: true

pkg/universe.dagger.io/go/test/container.cue

@@ -12,7 +12,10 @@ dagger.#Plan & {
 
 		simple: go.#Container & {
 			source: _source
-			command: args: ["version"]
+			command: {
+				name: "go"
+				args: ["version"]
+			}
 		}
 
 		override: {
@@ -23,7 +26,10 @@ dagger.#Plan & {
 			command: go.#Container & {
 				input:  base.output
 				source: _source
-				command: args: ["version"]
+				command: {
+					name: "go"
+					args: ["version"]
+				}
 			}
 		}
 	}

plan/task/clientfilesystemread.go

@@ -125,12 +125,9 @@ func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.
 		opts = append(opts, llb.IncludePatterns(dir.Include))
 	}
 
-	// Excludes .dagger directory by default
-	excludePatterns := []string{"**/.dagger/"}
 	if len(dir.Exclude) > 0 {
-		excludePatterns = dir.Exclude
+		opts = append(opts, llb.ExcludePatterns(dir.Exclude))
 	}
-	opts = append(opts, llb.ExcludePatterns(excludePatterns))
 
 	// FIXME: Remove the `Copy` and use `Local` directly.
 	//

plan/task/source.go

@@ -69,7 +69,7 @@ func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s *solv
 
 	lg.Debug().Str("path", path).Msg("loading local directory")
 	opts := []llb.LocalOption{
-		withCustomName(v, "Embed %s", path),
+		withCustomName(v, "Source %s", path),
 		llb.IncludePatterns(source.Include),
 		llb.ExcludePatterns(source.Exclude),
 		// Without hint, multiple `llb.Local` operations on the

source.cue

@@ -1,15 +0,0 @@
-package main
-
-import (
-	"dagger.io/dagger/core"
-)
-
-_source: core.#Source & {
-	path: "."
-	exclude: [
-		"ci",
-		"node_modules",
-		"cmd/dagger/dagger",
-		"cmd/dagger/dagger-debug",
-	]
-}