From 5539c33598e596aeda138d8e237e5b15235091bb Mon Sep 17 00:00:00 2001 From: divolgin Date: Mon, 9 Sep 2019 20:12:50 +0000 Subject: [PATCH] airgap upload --- Makefile | 4 +- ffi/airgap.go | 167 +++++++++++++++++++------------------ ffi/airgap_test.go | 51 ----------- ffi/online.go | 1 - go.mod | 1 + go.sum | 2 + pkg/image/image.go | 77 +++++++++++++++++ pkg/image/image_test.go | 62 ++++++++++++++ pkg/midstream/midstream.go | 6 +- pkg/pull/pull.go | 28 +++++-- 10 files changed, 257 insertions(+), 142 deletions(-) delete mode 100644 ffi/airgap_test.go create mode 100644 pkg/image/image.go create mode 100644 pkg/image/image_test.go diff --git a/Makefile b/Makefile index 07430952e0..e142173ebc 100644 --- a/Makefile +++ b/Makefile @@ -16,11 +16,11 @@ ffi: fmt vet .PHONY: fmt fmt: - go fmt ./pkg/... ./cmd/... + go fmt ./pkg/... ./cmd/... ./ffi/... .PHONY: vet vet: - go vet ./pkg/... ./cmd/... + go vet ./pkg/... ./cmd/... ./ffi/... .PHONY: gosec gosec: diff --git a/ffi/airgap.go b/ffi/airgap.go index 222ea662b6..ad75a89fef 100644 --- a/ffi/airgap.go +++ b/ffi/airgap.go @@ -5,14 +5,16 @@ import "C" import ( "archive/tar" "compress/gzip" - "os/exec" "fmt" "io" "io/ioutil" "net/http" "os" + "os/exec" "path/filepath" + "strings" + "github.com/docker/distribution/reference" "github.com/mholt/archiver" "github.com/pkg/errors" kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" @@ -21,21 +23,90 @@ import ( "k8s.io/client-go/kubernetes/scheme" ) -//export PullFromAirgap -func PullFromAirgap(licenseData string, airgapURL string, downstream string, outputFile string) int { - workspace, err := ioutil.TempDir("", "kots-airgap") +type ImageRef struct { + Domain string + Name string + Tag string + Digest string +} + +//export RewriteAndPushImageName +func RewriteAndPushImageName(imageFile, image, registryHost, registryOrg, username, password string) int { + imageRef, err := parseImageRef(image) if err != nil { - fmt.Printf("failed to create temp dir: %s\n", err) + fmt.Printf("failed to parse image %s: %s\n", image, err) + return 1 + } + localImage := imageRefToString(imageRef, registryHost, registryOrg) + + cmdArgs := []string{ + "copy", + "--dest-tls-verify=false", + } + if len(username) > 0 && len(password) > 0 { + cmdArgs = append(cmdArgs, fmt.Sprintf("--dest-creds=%s:%s", username, password)) + } + cmdArgs = append(cmdArgs, + fmt.Sprintf("oci-archive:%s", imageFile), + fmt.Sprintf("docker://%s", localImage), + ) + + cmd := exec.Command("skopeo", cmdArgs...) + cmdOutput, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("run failed with output: %s\n", cmdOutput) return 1 } - // defer os.RemoveAll(workspace) - // airgapDir contains release tar and all images as individual tars - airgapDir, err := downloadAirgapAchive(workspace, airgapURL) + return 0 +} + +func parseImageRef(image string) (*ImageRef, error) { + ref := &ImageRef{} + + // named, err := reference.ParseNormalizedNamed(image) + parsed, err := reference.ParseAnyReference(image) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse image name %q", image) + } + + if named, ok := parsed.(reference.Named); ok { + ref.Domain = reference.Domain(named) + ref.Name = named.Name() + } else { + return nil, errors.New(fmt.Sprintf("unsupported ref type: %T", parsed)) + } + + if tagged, ok := parsed.(reference.Tagged); ok { + ref.Tag = tagged.Tag() + } else if can, ok := parsed.(reference.Canonical); ok { + ref.Digest = can.Digest().String() + } else { + ref.Tag = "latest" + } + + return ref, nil +} + +func imageRefToString(ref *ImageRef, registryHost, registryOrg string) string { + pathParts := strings.Split(ref.Name, "/") + imageName := fmt.Sprintf("%s/%s", registryOrg, pathParts[len(pathParts)-1]) + + // there might be a way to do this with reference package too + if ref.Digest != "" { + return fmt.Sprintf("%s/%s@sha256:%s", registryHost, imageName, ref.Digest) + } + return fmt.Sprintf("%s/%s:%s", registryHost, imageName, ref.Tag) +} + +//export PullFromAirgap +func PullFromAirgap(licenseData, airgapDir, downstream, outputFile, registryHost, registryNamesapce string) int { + workspace, err := ioutil.TempDir("", "kots-airgap") if err != nil { - fmt.Printf("failed to download airgap archive: %s\n", err) + fmt.Printf("failed to create temp dir: %s\n", err) return 1 } + defer os.RemoveAll(workspace) // releaseDir is the contents of the release tar (yaml, no images) releaseDir, err := extractAppRelease(workspace, airgapDir) @@ -44,8 +115,6 @@ func PullFromAirgap(licenseData string, airgapURL string, downstream string, out return 1 } - ///............ - kotsscheme.AddToScheme(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode obj, _, err := decode([]byte(licenseData), nil, nil) @@ -73,7 +142,7 @@ func PullFromAirgap(licenseData string, airgapURL string, downstream string, out fmt.Printf("failed to create temp root path: %s\n", err.Error()) return 1 } - // defer os.RemoveAll(tmpRoot) + defer os.RemoveAll(tmpRoot) pullOptions := pull.PullOptions{ Downstreams: []string{downstream}, @@ -82,6 +151,11 @@ func PullFromAirgap(licenseData string, airgapURL string, downstream string, out ExcludeKotsKinds: true, RootDir: tmpRoot, ExcludeAdminConsole: true, + RewriteImages: pull.RewriteImages{ + ImageFiles: filepath.Join(airgapDir, "images"), + Host: registryHost, + Namespace: registryNamesapce, + }, } if _, err := pull.Pull(fmt.Sprintf("replicated://%s", license.Spec.AppSlug), pullOptions); err != nil { @@ -109,11 +183,6 @@ func PullFromAirgap(licenseData string, airgapURL string, downstream string, out return 1 } - if err := pushImages(filepath.Join(airgapDir, "images"), []string{}); err != nil { - fmt.Printf("unable to push images: %s\n", err.Error()) - return 1 - } - return 0 } @@ -256,68 +325,4 @@ func extractOneArchive(tgzFile string, destDir string) error { } return nil -} - -func pushImages(srcDir string, imageNameParts []string) error { - files, err := ioutil.ReadDir(srcDir) - if err != nil { - return errors.Wrapf(err, "failed to list image files") - } - - for _, file := range files { - if file.IsDir() { - // this function will modify the array argument - err := pushImages(filepath.Join(srcDir, file.Name()), append(imageNameParts, file.Name())) - if err != nil { - return errors.Wrapf(err, "failed to push images") - } - } else { - // this function will modify the array argument - if err := pushImageFromFile(filepath.Join(srcDir, file.Name()), append(imageNameParts, file.Name())); err != nil { - return errors.Wrapf(err, "failed to push image") - } - } - } - - return nil -} - -func pushImageFromFile(filename string, imageNameParts []string) error { - // TODO: don't hardcode registry name - imageName, err := imageNameFromNameParts("image-registry-lb:5000", imageNameParts) - if err != nil { - return errors.Wrapf(err, "failed to generate image name from %v", imageNameParts) - } - cmd := exec.Command("skopeo", "copy", "--dest-tls-verify=false", fmt.Sprintf("oci-archive:%s", filename), fmt.Sprintf("docker://%s", imageName)) - stdoutStderr, err := cmd.CombinedOutput() - if err != nil { - fmt.Printf("run failed with output: %s\n", stdoutStderr) - return errors.Wrap(err, "failed to execute skopeo") - } - - return nil -} - -func imageNameFromNameParts(registry string, nameParts []string) (string, error) { - // imageNameParts looks like this: - // ["quay.io", "someorg", "imagename", "imagetag"] - // or - // ["quay.io", "someorg", "imagename", "sha256", ""] - // we want to replace host with local registry and build image name from the remaining parts - - if len(nameParts) < 4 { - return "", fmt.Errorf("not enough parts in image name: %v", nameParts) - } - - var name, tag string - nameParts[0] = registry - if nameParts[len(nameParts) - 2] == "sha256" { - tag = fmt.Sprintf("@sha256:%s", nameParts[len(nameParts) - 1]) - name = filepath.Join(nameParts[:len(nameParts)-2]...) - } else { - tag = fmt.Sprintf(":%s", nameParts[len(nameParts) - 1]) - name = filepath.Join(nameParts[:len(nameParts)-1]...) - } - - return fmt.Sprintf("%s%s", name, tag), nil } \ No newline at end of file diff --git a/ffi/airgap_test.go b/ffi/airgap_test.go deleted file mode 100644 index 29d35eacb9..0000000000 --- a/ffi/airgap_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package main - -import ( - "testing" - "fmt" - - "github.com/stretchr/testify/assert" -) - -func Test_ImageNameFromNameParts(t *testing.T) { - registry := "localhost:5000" - - tests := []struct { - name string - parts []string - expected string - isError bool - }{ - { - name: "bad name format", - parts: []string{"quay.io", "debian", "latest"}, - expected: "", - isError: true, - }, - { - name: "four parts with tag", - parts: []string{"quay.io", "someorg", "debian", "latest"}, - expected: fmt.Sprintf("%s/someorg/debian:latest", registry), - isError: false, - }, - { - name: "five parts with sha", - parts: []string{"quay.io", "someorg", "debian", "sha256", "1234567890abcdef"}, - expected: fmt.Sprintf("%s/someorg/debian@sha256:1234567890abcdef", registry), - isError: false, - }, - } - - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - url, err := imageNameFromNameParts(registry, test.parts) - if test.isError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.Equal(t, test.expected, url) - } - }) - } -} diff --git a/ffi/online.go b/ffi/online.go index 7cb742278d..4c1ec16757 100644 --- a/ffi/online.go +++ b/ffi/online.go @@ -80,4 +80,3 @@ func PullFromLicense(licenseData string, downstream string, outputFile string) i return 0 } - diff --git a/go.mod b/go.mod index e51f85f381..0b00019905 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/cjbassi/drawille-go v0.0.0-20190126131713-27dc511fe6fd // indirect github.com/cloudflare/cfssl v0.0.0-20190808011637-b1ec8c586c2a // indirect github.com/cyphar/filepath-securejoin v0.2.2 // indirect + github.com/docker/distribution v2.7.1+incompatible github.com/docker/go-units v0.4.0 github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 // indirect github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 // indirect diff --git a/go.sum b/go.sum index 393a42bfa9..54b3c0f230 100644 --- a/go.sum +++ b/go.sum @@ -131,6 +131,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.0.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dnaeon/go-vcr v0.0.0-20180920040454-5637cf3d8a31/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= @@ -505,6 +506,7 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= diff --git a/pkg/image/image.go b/pkg/image/image.go new file mode 100644 index 0000000000..b489ef9c1f --- /dev/null +++ b/pkg/image/image.go @@ -0,0 +1,77 @@ +package image + +import ( + "fmt" + "io/ioutil" + "path" + "path/filepath" + + "github.com/pkg/errors" + "sigs.k8s.io/kustomize/v3/pkg/image" +) + +func BuildRewriteList(rootDir string, host string, namespace string) ([]image.Image, error) { + images, err := findImages(rootDir, host, namespace, []string{}) + return images, errors.Wrap(err, "failed to find images") +} + +func findImages(srcDir string, host string, namespace string, imageNameParts []string) ([]image.Image, error) { + files, err := ioutil.ReadDir(srcDir) + if err != nil { + return nil, errors.Wrapf(err, "failed to list image files") + } + + images := make([]image.Image, 0) + for _, file := range files { + if file.IsDir() { + moreImages, err := findImages(filepath.Join(srcDir, file.Name()), host, namespace, append(imageNameParts, file.Name())) + if err != nil { + return nil, err // no error wrapping because this is a recursive call + } + images = append(images, moreImages...) + } else { + image, err := imageInfoFromFile(host, namespace, append(imageNameParts, file.Name())) + if err != nil { + return nil, errors.Wrapf(err, "failed to create local image name") + } + images = append(images, image) + } + } + + return images, nil +} + +func imageInfoFromFile(registryHost string, namespace string, nameParts []string) (image.Image, error) { + // imageNameParts looks like this: + // ["quay.io", "someorg", "imagename", "imagetag"] + // or + // ["quay.io", "someorg", "imagename", "sha256", ""] + // we want to discard everything upto "imagename" and replace that with local host and namespace + + image := image.Image{} + + if len(nameParts) < 4 { + return image, fmt.Errorf("not enough parts in image name: %v", nameParts) + } + + newImageNameParts := []string{registryHost, namespace} + var originalName, tag, separator string + if nameParts[len(nameParts)-2] == "sha256" { + newImageNameParts = append(newImageNameParts, nameParts[len(nameParts)-3]) + originalName = path.Join(nameParts[:len(nameParts)-2]...) + tag = fmt.Sprintf("sha256:%s", nameParts[len(nameParts)-1]) + separator = "@" + image.Digest = nameParts[len(nameParts)-1] + } else { + newImageNameParts = append(newImageNameParts, nameParts[len(nameParts)-2]) + originalName = path.Join(nameParts[:len(nameParts)-1]...) + tag = fmt.Sprintf("%s", nameParts[len(nameParts)-1]) + separator = ":" + image.NewTag = tag + } + + image.Name = fmt.Sprintf("%s%s%s", originalName, separator, tag) + image.NewName = path.Join(newImageNameParts...) + + return image, nil +} diff --git a/pkg/image/image_test.go b/pkg/image/image_test.go new file mode 100644 index 0000000000..315fb76f1e --- /dev/null +++ b/pkg/image/image_test.go @@ -0,0 +1,62 @@ +package image + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/v3/pkg/image" +) + +func Test_ImageNameFromNameParts(t *testing.T) { + registry := "localhost:5000" + namespace := "somebigbank" + + tests := []struct { + name string + parts []string + expected image.Image + isError bool + }{ + { + name: "bad name format", + parts: []string{"quay.io", "debian", "latest"}, + expected: image.Image{}, + isError: true, + }, + { + name: "four parts with tag", + parts: []string{"quay.io", "someorg", "debian", "0.1"}, + expected: image.Image{ + Name: "quay.io/someorg/debian:0.1", + NewName: fmt.Sprintf("%s/%s/debian", registry, namespace), + NewTag: "0.1", + Digest: "", + }, + isError: false, + }, + { + name: "five parts with sha", + parts: []string{"quay.io", "someorg", "debian", "sha256", "1234567890abcdef"}, + expected: image.Image{ + Name: "quay.io/someorg/debian@sha256:1234567890abcdef", + NewName: fmt.Sprintf("%s/%s/debian", registry, namespace), + NewTag: "", + Digest: "1234567890abcdef", + }, + isError: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + image, err := imageInfoFromFile(registry, namespace, test.parts) + if test.isError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, test.expected, image) + } + }) + } +} diff --git a/pkg/midstream/midstream.go b/pkg/midstream/midstream.go index 1c55a30a19..8395346280 100644 --- a/pkg/midstream/midstream.go +++ b/pkg/midstream/midstream.go @@ -2,6 +2,7 @@ package midstream import ( "github.com/replicatedhq/kots/pkg/base" + "sigs.k8s.io/kustomize/v3/pkg/image" kustomizetypes "sigs.k8s.io/kustomize/v3/pkg/types" ) @@ -10,13 +11,14 @@ type Midstream struct { Base *base.Base } -func CreateMidstream(b *base.Base) (*Midstream, error) { +func CreateMidstream(b *base.Base, images []image.Image) (*Midstream, error) { kustomization := kustomizetypes.Kustomization{ TypeMeta: kustomizetypes.TypeMeta{ APIVersion: "kustomize.config.k8s.io/v1beta1", Kind: "Kustomization", }, - Bases: []string{}, + Bases: []string{}, + Images: images, } m := Midstream{ diff --git a/pkg/pull/pull.go b/pkg/pull/pull.go index 69652c9eb8..440c4e8f6e 100644 --- a/pkg/pull/pull.go +++ b/pkg/pull/pull.go @@ -3,17 +3,19 @@ package pull import ( "io/ioutil" "net/url" - "path" + "path/filepath" "github.com/pkg/errors" kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" kotsscheme "github.com/replicatedhq/kots/kotskinds/client/kotsclientset/scheme" "github.com/replicatedhq/kots/pkg/base" "github.com/replicatedhq/kots/pkg/downstream" + kotsimage "github.com/replicatedhq/kots/pkg/image" "github.com/replicatedhq/kots/pkg/logger" "github.com/replicatedhq/kots/pkg/midstream" "github.com/replicatedhq/kots/pkg/upstream" "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/kustomize/v3/pkg/image" ) type PullOptions struct { @@ -28,6 +30,13 @@ type PullOptions struct { SharedPassword string CreateAppDir bool Silent bool + RewriteImages RewriteImages +} + +type RewriteImages struct { + ImageFiles string + Host string + Namespace string } // PullApplicationMetadata will return the application metadata yaml, if one is @@ -138,15 +147,24 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { return "", errors.Wrap(err, "failed to write base") } + var images []image.Image + if pullOptions.LocalPath != "" { + i, err := kotsimage.BuildRewriteList(pullOptions.RewriteImages.ImageFiles, pullOptions.RewriteImages.Host, pullOptions.RewriteImages.Namespace) + if err != nil { + return "", errors.Wrap(err, "failed to rewrite images") + } + images = i + } + log.ActionWithSpinner("Creating midstream") - m, err := midstream.CreateMidstream(b) + m, err := midstream.CreateMidstream(b, images) if err != nil { return "", errors.Wrap(err, "failed to create midstream") } log.FinishSpinner() writeMidstreamOptions := midstream.WriteOptions{ - MidstreamDir: path.Join(b.GetOverlaysDir(writeBaseOptions), "midstream"), + MidstreamDir: filepath.Join(b.GetOverlaysDir(writeBaseOptions), "midstream"), BaseDir: u.GetBaseDir(writeUpstreamOptions), } if err := m.WriteMidstream(writeMidstreamOptions); err != nil { @@ -161,7 +179,7 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { } writeDownstreamOptions := downstream.WriteOptions{ - DownstreamDir: path.Join(b.GetOverlaysDir(writeBaseOptions), "downstreams", downstreamName), + DownstreamDir: filepath.Join(b.GetOverlaysDir(writeBaseOptions), "downstreams", downstreamName), MidstreamDir: writeMidstreamOptions.MidstreamDir, } @@ -178,7 +196,7 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { } } - return path.Join(pullOptions.RootDir, u.Name), nil + return filepath.Join(pullOptions.RootDir, u.Name), nil } func parseLicenseFromFile(filename string) (*kotsv1beta1.License, error) {