Merge pull request #628 from TomChv/improve-docker-push

Improve docker.#Push definition
This commit is contained in:
Andrea Luzzardi 2021-07-01 18:25:40 +02:00 committed by GitHub
commit e020b23649
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 447 additions and 131 deletions

View File

@ -70,18 +70,23 @@ _No output._
## docker.#Push
Push a docker image
Push a docker image to a remote registry
### docker.#Push Inputs
| Name | Type | Description |
| ------------- |:-------------: |:-------------: |
|*ref* | `string` |Remote ref (example: "index.docker.io/alpine:latest") |
|*source* | `dagger.#Artifact` |Image |
| Name | Type | Description |
| ------------- |:-------------: |:-------------: |
|*target* | `string` |Remote target (example: "index.docker.io/alpine:latest") |
|*source* | `dagger.#Artifact` |Image source |
|*auth.username* | `string` |Username |
|*auth.secret* | `string` |Password or secret |
### docker.#Push Outputs
_No output._
| Name | Type | Description |
| ------------- |:-------------: |:-------------: |
|*ref* | `string` |Image ref |
|*digest* | `string` |Image digest |
## docker.#Run

View File

@ -2,10 +2,10 @@ package solver
import (
"context"
"net/url"
"strings"
"sync"
"github.com/docker/distribution/reference"
bkauth "github.com/moby/buildkit/session/auth"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
@ -40,9 +40,9 @@ func (a *RegistryAuthProvider) Register(server *grpc.Server) {
}
func (a *RegistryAuthProvider) Credentials(ctx context.Context, req *bkauth.CredentialsRequest) (*bkauth.CredentialsResponse, error) {
reqURL, err := parseAuthHost(req.Host)
if err != nil {
return nil, err
host := req.Host
if host == "registry-1.docker.io" {
host = "docker.io"
}
a.m.RLock()
@ -54,7 +54,7 @@ func (a *RegistryAuthProvider) Credentials(ctx context.Context, req *bkauth.Cred
return nil, err
}
if u.Host == reqURL.Host {
if u == host {
return auth, nil
}
}
@ -62,15 +62,16 @@ func (a *RegistryAuthProvider) Credentials(ctx context.Context, req *bkauth.Cred
return &bkauth.CredentialsResponse{}, nil
}
func parseAuthHost(host string) (*url.URL, error) {
if host == "registry-1.docker.io" {
host = "https://index.docker.io/v1/"
}
func parseAuthHost(host string) (string, error) {
host = strings.TrimPrefix(host, "http://")
host = strings.TrimPrefix(host, "https://")
if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
host = "https://" + host
ref, err := reference.ParseNormalizedNamed(host)
if err != nil {
return "", err
}
return url.Parse(host)
return reference.Domain(ref), nil
}
func (a *RegistryAuthProvider) FetchToken(ctx context.Context, req *bkauth.FetchTokenRequest) (rr *bkauth.FetchTokenResponse, err error) {

View File

@ -0,0 +1,2 @@
# dagger state
state/**

View File

@ -0,0 +1,27 @@
plan:
module: ./docker
package: ./tests/pull
name: docker-pull
inputs:
ref:
text: docker.io/daggerio/ci-test:xtyzsocvpici@sha256:35fc94d52b4fa53c2caa38ff11e13182e6f88c651eb0846728d1007d931f0d3c
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age:
- recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0WFI2ZGFUOWgvNkdlZ3Na
dEE5dTlVQi8vUVJqcHUxWE9GSmdnNmZLMHhRCm1sbFlJbEw1ZVFSVXU4MCtkT09l
dVR1WE5XUkVpSXA3aXN5TzZLaWJRNnMKLS0tIDZINGpzODdXVUdKVVpFMjFUbUFO
SG1raUVNTzZIWDltV1pOS3hySHlJeWcKg3blmstOGcxtPww513+mAEA0MWOXwNAT
5ngRvG6MraW3g9dhIuUYOwjuJyz1Z07/DBEocSxnjSyw45ZCkM1/9Q==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2021-06-26T14:52:40Z"
mac: ENC[AES256_GCM,data:IVEK6NFWEmNv8kRay2wVNhrsXVazVinIYRDLy7DTvaiWXyQYun//joK3QIoKz3dqi9rXeuTd95B13RxVQWKy/8cpmryg4QCwAaCj8erb5FHMRfn5/mAAV3NL5oAoOpKF4lZByrfdrXTJKppGWwYOFy8X693kK3FUzoUpIW2OqXg=,iv:qinwsUefQ7M+0OCTISPdQ9q//xsPitmHeCpdF00BJoo=,tag:reHZ5j0nz9fjAEFpR7IGGQ==,type:str]
pgp: []
encrypted_suffix: secret
version: 3.7.1

View File

@ -0,0 +1,2 @@
# dagger state
state/**

View File

@ -0,0 +1,29 @@
plan:
module: ./docker
package: ./tests/push-invalid-creds
name: docker-push-invalid-creds
inputs:
TestRegistry.secret:
text: ENC[AES256_GCM,data:PckymCtA/Q==,iv:to7XhUUcZrWDga7uT4C067BRzHEzmTPDUNAEb2TpS/I=,tag:jUTk8uGd185hmIvi/IHpww==,type:str]
TestRegistry.username:
text: invalid
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age:
- recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4czBwNGtSdGZqdEZ5WDlM
SHVYOU5zdFl4L2ptZk5rSHgwek1aaDNicENFCkJ4OUIweU5OZTVKalpTSkhYaGxB
RUpHZmVvU3g3Y2tBZnRUcHh0TE52M1EKLS0tIHI1VUt1aUR0a0tDNHJVTHY4eEt1
VC8wSTZvUE5UaDg2WE1CaGMzR3M1TEkK9v83AVI4lvFgjKCg8UmQrcxarlESWTfV
2cDdWgoH7ZqgXo5jFv2tn8qQWHKl8eTTeYUWn8GoNVPKrCroax2fiQ==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2021-06-26T15:00:53Z"
mac: ENC[AES256_GCM,data:ptE3WydZDuethnN5Qh26uAfndRbT+RKz2mktH4s2KyRNeDKgiBfwOVS1xoTxz+nkFoms0Cxac3iaVwZLpZXniQUbOAYY1fzfmyL32bfAUdNFs7P6K0thwSy8r8LJ38GvxHzZW289YVFTGSaJWCapbrcGzl6B7Aj5RcQ+Hhu32K8=,iv:PA2R7Q8y8F//RGnHpOCmxp8jWKXlAZ3Yfo0xbtPfx2E=,tag:EmxBTb9WVrDdOmgDHEDYfg==,type:str]
pgp: []
encrypted_suffix: secret
version: 3.7.1

View File

@ -0,0 +1,2 @@
# dagger state
state/**

View File

@ -0,0 +1,35 @@
plan:
module: ./docker
package: ./tests/push-multi-registry
name: docker-push-multi-registry
inputs:
TestRemoteAWS.awsConfig.accessKey:
secret: ENC[AES256_GCM,data:Vg+RRHYV5p0twlKtq0zGzokTsXY=,iv:XXEjaZBmS7A+KBZQ/0ZJ4WLH3M5dthg0lq86BhHOt2U=,tag:q6QSQkH9Jz/e4FGlBNllug==,type:str]
TestRemoteAWS.awsConfig.region:
text: us-east-2
TestRemoteAWS.awsConfig.secretKey:
secret: ENC[AES256_GCM,data:uk5BBJhmc8RadT1FSIsnW+/Rvs8c+kIhshBia+DX+UEWiuPV+RwXfw==,iv:DzXcvUcy3amU7wCA6XFgPvGUAU+dxPZQMHKM94d9PlY=,tag:QDDs4kg6cFPLLGRM6sHzfg==,type:str]
TestRemoteDocker.dockerConfig.secret:
secret: ENC[AES256_GCM,data:bxlKdGBSd2Rxf0Kmw8+QO1h0308rGYPqzUO17Eg4RUh2WQjd,iv:vNZww3t8yBrcmmddJghtJWfkz3G9j2CPGyx9B3e/WK8=,tag:qlUl2dkREcGZxdKeAzPjzQ==,type:str]
TestRemoteDocker.dockerConfig.username:
text: daggertest
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age:
- recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnc1JZMUpTUnFZNVI3RHh3
a0RtTDNIQ2xjaHJlM0tNbndta1NlMjc2a2dFCkQzV0FpMFBHZUdZb0RMYW1DUGN2
TlZVQ2dhdWt3OEN5LzZEYXR4QkFSTTAKLS0tIFdaS0Z0dG85QXNNTnpXZ0dFUGpY
Yy81dlJ5cDhCL1VCc0szSE9Dbjh0TUEK+xcj1bHhJr0MR+2QLL0Y+at0/SFXcutx
VpUkCykV3eBV6P9I51+3NeJ/ZMmJ43N2geFFJNeacmn8uQKNxpgGGw==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2021-07-01T13:15:01Z"
mac: ENC[AES256_GCM,data:vpU0XJA/H/ra4BIuReWJAOLvFW4s+xHDAxxSYoU1WkdX68EUb1jbuhEqyDqlhQRn6lf3qSt9kbnbiiw39/mrdBFnwDg5DLjOPT17G/rBiSp9p+1e4mN8hGNp79uen+dDQX7f4NSxZ4nroMVtEuIuBrbFaZUMYVaBYEHjGuw2hgc=,iv:/nW7lpopSsqTwoaPgiHrabtl8aOZtJEezkwBDqi15Tg=,tag:uw3Hj+/t3Y5U0wpK7g+tJg==,type:str]
pgp: []
encrypted_suffix: secret
version: 3.7.1

View File

@ -0,0 +1,2 @@
# dagger state
state/**

View File

@ -0,0 +1,29 @@
plan:
module: ./docker
package: ./tests/push
name: docker-push
inputs:
TestRegistry.secret:
secret: ENC[AES256_GCM,data:ooc+0IjYtX9tkM7q1i4Ws6CorZsWtGQzHbjGx+j892iTZC7Q,iv:asdJzuRAHBRhD/FlkEd1VvX1tIz/qupBL7sMQWxZL5E=,tag:yuTyDx7hZeC+cmHx6tspmQ==,type:str]
TestRegistry.username:
text: daggertest
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age:
- recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrcnpRZ203QzhtKzc3bzF4
Y002V0JUUnNZMks2VS83SjBOdVZid1dxbTJjCmc5VGtvM3lOejEvQ3VMZ1ZyZElZ
Skd3ZWxRMHdQRHdtZFBYUFMweDFlL28KLS0tIHhHeUh4a2gvb2w3UTEyNFZaK0dS
UjFJYTc1UUUzSFVkZjQ2blRsSGpVdVEKOanMR3+WlAgoDfqTUW7WPW1ytT3NdkTX
4Rqo49QmnuKFJ9tKoBFQOqgIo8E/lpcOkeIUiy5e/35FvsZ/KFk/pg==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2021-06-26T14:53:13Z"
mac: ENC[AES256_GCM,data:kxzdmyQwRIVP9D+w9LzRDIZOxDUqpSaGeD/GqaVxnQivEUjVFK5ePAcYV1fzjU4eeO3chIrP8NpvpIKZ1LztQddsPdTj72Yffgc5yq8/dGa3njiM8p9oa5hnZNoxLtyVPgRoNy3ZUZ6YSN9nqCFWW0DCjeSoiDlPX+1Vj/S6PeM=,iv:DhX4N6idS3VBaqau6k9yH+li34hOd3jqBsJJQu/P3Jw=,tag:X+taa7XgRmovR0JkhGpi7g==,type:str]
pgp: []
encrypted_suffix: secret
version: 3.7.1

View File

@ -59,7 +59,7 @@ package op
#DockerLogin: {
do: "docker-login"
target: string | *"https://index.docker.io/v1/"
target: string
username: string
// FIXME: should be a #Secret (circular import)
secret: string | bytes

View File

@ -28,18 +28,64 @@ import (
]
}
// Push a docker image
// Push a docker image to a remote registry
#Push: {
// Remote ref (example: "index.docker.io/alpine:latest")
ref: string @dagger(input)
// Remote target (example: "index.docker.io/alpine:latest")
target: string @dagger(input)
// Image
// Image source
source: dagger.#Artifact @dagger(input)
#up: [
op.#Load & {from: source},
op.#PushContainer & {"ref": ref},
// Registry auth
auth: {
// Username
username: string @dagger(input)
// Password or secret
secret: string @dagger(input)
}
push: #up: [
op.#Load & {from: source},
if auth != _|_ {
op.#DockerLogin & {
"target": target
username: auth.username
secret: auth.secret
}
},
op.#PushContainer & {ref: target},
op.#Subdir & {dir: "/dagger"},
]
// Image ref
ref: {
string
#up: [
op.#Load & {from: push},
op.#Export & {
source: "/image_ref"
},
]
} @dagger(output)
// Image digest
digest: {
string
#up: [
op.#Load & {from: push},
op.#Export & {
source: "/image_digest"
},
]
} @dagger(output)
}
#Run: {

View File

@ -0,0 +1,25 @@
package docker
import (
"alpha.dagger.io/dagger/op"
"alpha.dagger.io/alpine"
)
ref: string @dagger(input)
TestPull: {
pull: #Pull & {from: ref}
check: #up: [
op.#Load & {from: alpine.#Image},
op.#Exec & {
always: true
args: [
"sh", "-c", """
grep -q "test" /src/test.txt
""",
]
mount: "/src": from: pull
},
]
}

View File

@ -0,0 +1,35 @@
package docker
import (
"alpha.dagger.io/random"
)
TestRegistry: {
username: string @dagger(input)
secret: string @dagger(input)
}
TestPush: {
// Generate a random string
// Seed is used to force buildkit execution and not simply use a previous generated string.
tag: random.#String & {seed: "docker push and pull should fail"}
target: "daggerio/ci-test:\(tag.out)"
image: #ImageFromDockerfile & {
dockerfile: """
FROM alpine
RUN echo "test" > /test.txt
"""
context: ""
}
push: #Push & {
"target": target
source: image
auth: {
username: TestRegistry.username
secret: TestRegistry.secret
}
}
}

View File

@ -0,0 +1,93 @@
package docker
import (
"alpha.dagger.io/aws"
"alpha.dagger.io/aws/ecr"
"alpha.dagger.io/dagger"
"alpha.dagger.io/dagger/op"
"alpha.dagger.io/random"
"alpha.dagger.io/alpine"
)
//
// /!\ README /!\
// The objective is to push an image on multiple registries to verify
// that we correctly handle that kind of configuration
//
TestResources: {
// Generate a random string
// Seed is used to force buildkit execution and not simply use a previous generated string.
suffix: random.#String & {seed: "docker multi registry"}
image: #ImageFromDockerfile & {
dockerfile: """
FROM alpine
RUN echo "test" > /test.txt
"""
context: ""
}
}
TestRemoteAWS: {
awsConfig: aws.#Config
ecrCreds: ecr.#Credentials & {
config: awsConfig
}
target: "125635003186.dkr.ecr.\(awsConfig.region).amazonaws.com/dagger-ci:test-ecr-\(TestResources.suffix.out)"
remoteImg: #Push & {
"target": target
source: TestResources.image
auth: {
username: ecrCreds.username
secret: ecrCreds.secret
}
}
}
#TestGetSecret: {
secret: dagger.#Artifact
out: {
string
#up: [
op.#Load & {from: alpine.#Image},
op.#Exec & {
always: true
args: ["sh", "-c", "cp /input/secret /secret"]
mount: "/input/secret": "secret": secret
},
op.#Export & {
source: "/secret"
},
]
}
}
TestRemoteDocker: {
dockerConfig: {
username: string & dagger.#Input
secret: dagger.#Secret & dagger.#Input
}
secret: #TestGetSecret & {
secret: dockerConfig.secret
}
target: "daggerio/ci-test:test-docker-\(TestResources.suffix.out)"
remoteImg: #Push & {
"target": target
source: TestResources.image
auth: {
username: dockerConfig.username
"secret": secret.out
}
}
}

View File

@ -0,0 +1,64 @@
package docker
import (
"alpha.dagger.io/dagger/op"
"alpha.dagger.io/dagger"
"alpha.dagger.io/alpine"
"alpha.dagger.io/random"
)
TestRegistry: {
username: string @dagger(input)
secret: dagger.#Secret @dagger(input)
}
#TestGetSecret: {
secret: dagger.#Artifact
out: {
string
#up: [
op.#Load & {from: alpine.#Image},
op.#Exec & {
always: true
args: ["sh", "-c", "cp /input/secret /secret"]
mount: "/input/secret": "secret": secret
},
op.#Export & {
source: "/secret"
},
]
}
}
TestPush: {
// Generate a random string
// Seed is used to force buildkit execution and not simply use a previous generated string.
tag: random.#String & {seed: "docker push"}
target: "daggerio/ci-test:\(tag.out)"
secret: #TestGetSecret & {
secret: TestRegistry.secret
}
image: #ImageFromDockerfile & {
dockerfile: """
FROM alpine
RUN echo "test" > /test.txt
"""
context: ""
}
push: #Push & {
"target": target
source: image
auth: {
username: TestRegistry.username
"secret": secret.out
}
}
}

View File

@ -62,6 +62,29 @@ setup() {
dagger -e docker-build up
}
@test "docker push and pull" {
skip "An occasional data race condition happen in the CI. Must be fix before execute that test"
# Push image
dagger -e docker-push up
# Get image reference
dagger -e docker-pull input text ref "$(dagger -e docker-push query -c TestPush.push.ref | tr -d '\n' | tr -d '\"')"
# Pull image
dagger -e docker-pull up
}
@test "docker push: multi registry" {
skip "An occasional data race condition happen in the CI. Must be fix before execute that test"
run dagger -e docker-push-multi-registry up
}
@test "docker push: invalid credential" {
# Push image (SHOULD FAIL)
run dagger -e docker-push-invalid-creds up
assert_failure
}
@test "docker command: ssh" {
dagger -e docker-command-ssh up
}

View File

@ -21,6 +21,7 @@ TestPushContainer: {
ref: "daggerio/ci-test:\(tag.out)"
#up: [
op.#DockerLogin & {
target: ref
registry
},
op.#WriteFile & {

View File

@ -1,16 +0,0 @@
setup() {
load 'helpers'
common_setup
}
# FIXME: move to universe/universe.bats
# Assigned to: <ADD YOUR NAME HERE>
# Changes in https://github.com/dagger/dagger/pull/628
@test "stdlib: docker: push-and-pull" {
skip_unless_secrets_available "$TESTDIR"/stdlib/docker/push-pull/inputs.yaml
# check that they succeed with the credentials
run "$DAGGER" compute --input-yaml "$TESTDIR"/stdlib/docker/push-pull/inputs.yaml --input-dir source="$TESTDIR"/stdlib/docker/push-pull/testdata "$TESTDIR"/stdlib/docker/push-pull/
assert_success
}

View File

@ -1,23 +0,0 @@
registry:
username: ENC[AES256_GCM,data:YDDLkr32orAgQw==,iv:ezThCQJv+bVBf8SdfSa2HFoP+eu6IZMPl5xvMOGDcps=,tag:sEV9Sonc9rjDbxXsV+UBIA==,type:str]
secret: ENC[AES256_GCM,data:moBq7PwFdtL/Z58ez+V1gR8QJsFRZEMsF82H/W6aJgf8Xdw8,iv:YAXcRzBoemmef5PBdAOBa5acNPo4BoKH7Ngud/CWYfA=,tag:LFkJvUZdltgHJ8TKVEeS/Q==,type:str]
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age:
- recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGVmQxTXNSeU1scWJvVDFJ
cExOL3AvR1JRRWp0cFFRWGtvQ1VKc2t1SUVFClVCS1hpN1dNTktoaWZ3R09OMFVM
STRyWmtHRVROMW1Oa28yQkMwOHd1UUUKLS0tIE5LL1pEb1dMSEVXTHBsNlJxOTcr
U2FyQUtYcXVVVTlVcW5zRXh5aUk3RUUKGiWb9jSl5xRHQxB56LtNclV5Jhs50sS7
SAOBWgaYPjLpsI1oxgXf+B1FgBUEt3EMccrWRW85VvnOKOAUAJ53pQ==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2021-05-28T20:49:27Z"
mac: ENC[AES256_GCM,data:we6IaVqfT6KZ4s97JbdFCbxL2zotojLRLEbmgwEAfBhz4KAitulRItMn4I6aD1dEIwYGAFtQEcf+Wqz2yT7JC6iz1s2zNtGIaMbxxQZD6EQcJvNmY3vzqC4SKf0cRENGZWI5OscH9VVenTmOAxwwWvp9W4J52d2w9FAD9+vCl/c=,iv:vf8mZwr+z7DjCVHaRbk8jQO9/pso5INy/FmCPq/xlzo=,tag:sgSvlksSOVq5LU0ycAsXxw==,type:str]
pgp: []
unencrypted_suffix: _unencrypted
version: 3.7.1

View File

@ -1,65 +0,0 @@
package main
import (
"alpha.dagger.io/dagger"
"alpha.dagger.io/dagger/op"
"alpha.dagger.io/alpine"
"alpha.dagger.io/docker"
"alpha.dagger.io/random"
)
source: dagger.#Artifact
registry: {
username: string
secret: string
}
TestPushAndPull: {
tag: random.#String & {
seed: ""
}
ref: "daggerio/ci-test:\(tag.out)"
// Create image
image: docker.#ImageFromDockerfile & {
dockerfile: """
FROM alpine
COPY test.txt /test.txt
"""
context: source
}
// Login
login: #up: [
op.#DockerLogin & {
registry
},
]
// Push image
push: docker.#Push & {
"ref": ref
source: image
}
// Push image
pull: docker.#Pull & {
from: push.ref
}
// Check the content
verify: #up: [
op.#Load & {from: alpine.#Image},
op.#Exec & {
always: true
args: [
"sh", "-c", """
grep -q "test" /src/test.txt
""",
]
mount: "/src": from: pull
},
]
}

View File

@ -1 +0,0 @@
test