From ad685e67c9209d868ef3af24cd3bc4687a8ffef7 Mon Sep 17 00:00:00 2001 From: Carolyn Van Slyck Date: Wed, 6 Apr 2022 13:47:30 -0500 Subject: [PATCH 1/5] Pass context.Context through to package client Update package client to support structured logging before making changes to that code in subsequent commits. Signed-off-by: Carolyn Van Slyck --- cmd/porter/mixins.go | 2 +- cmd/porter/plugins.go | 2 +- pkg/pkgmgmt/client/helpers.go | 3 +- pkg/pkgmgmt/client/install.go | 78 ++++++++++++++++-------------- pkg/pkgmgmt/client/install_test.go | 9 ++-- pkg/pkgmgmt/feed/feed.go | 7 ++- pkg/pkgmgmt/pkgmgmt.go | 3 +- pkg/porter/mixins.go | 5 +- pkg/porter/mixins_test.go | 3 +- pkg/porter/plugins.go | 5 +- pkg/porter/plugins_test.go | 3 +- 11 files changed, 68 insertions(+), 52 deletions(-) diff --git a/cmd/porter/mixins.go b/cmd/porter/mixins.go index 118d37259..044e50cfd 100644 --- a/cmd/porter/mixins.go +++ b/cmd/porter/mixins.go @@ -94,7 +94,7 @@ By default mixins are downloaded from the official Porter mixin feed at https:// return opts.Validate(args) }, RunE: func(cmd *cobra.Command, args []string) error { - return p.InstallMixin(opts) + return p.InstallMixin(cmd.Context(), opts) }, } diff --git a/cmd/porter/plugins.go b/cmd/porter/plugins.go index ef787fbd6..fc1b34a62 100644 --- a/cmd/porter/plugins.go +++ b/cmd/porter/plugins.go @@ -114,7 +114,7 @@ By default plugins are downloaded from the official Porter plugin feed at https: return opts.Validate(args) }, RunE: func(cmd *cobra.Command, args []string) error { - return p.InstallPlugin(opts) + return p.InstallPlugin(cmd.Context(), opts) }, } diff --git a/pkg/pkgmgmt/client/helpers.go b/pkg/pkgmgmt/client/helpers.go index 111079cc4..5d6b21171 100644 --- a/pkg/pkgmgmt/client/helpers.go +++ b/pkg/pkgmgmt/client/helpers.go @@ -1,6 +1,7 @@ package client import ( + "context" "fmt" "path" "testing" @@ -40,7 +41,7 @@ func (p *TestPackageManager) GetMetadata(name string) (pkgmgmt.PackageMetadata, return nil, fmt.Errorf("%s %s not installed", p.PkgType, name) } -func (p *TestPackageManager) Install(o pkgmgmt.InstallOptions) error { +func (p *TestPackageManager) Install(ctx context.Context, opts pkgmgmt.InstallOptions) error { // do nothing return nil } diff --git a/pkg/pkgmgmt/client/install.go b/pkg/pkgmgmt/client/install.go index 5413ce966..a9c0761e6 100644 --- a/pkg/pkgmgmt/client/install.go +++ b/pkg/pkgmgmt/client/install.go @@ -1,6 +1,7 @@ package client import ( + "context" "encoding/json" "fmt" "io" @@ -13,45 +14,47 @@ import ( "get.porter.sh/porter/pkg" "get.porter.sh/porter/pkg/pkgmgmt" "get.porter.sh/porter/pkg/pkgmgmt/feed" - "github.com/pkg/errors" + "get.porter.sh/porter/pkg/tracing" ) const PackageCacheJSON string = "cache.json" -func (fs *FileSystem) Install(opts pkgmgmt.InstallOptions) error { +func (fs *FileSystem) Install(ctx context.Context, opts pkgmgmt.InstallOptions) error { var err error if opts.FeedURL != "" { - err = fs.InstallFromFeedURL(opts) + err = fs.InstallFromFeedURL(ctx, opts) } else { - err = fs.InstallFromURL(opts) + err = fs.InstallFromURL(ctx, opts) } if err != nil { return err } - return fs.savePackageInfo(opts) + return fs.savePackageInfo(ctx, opts) } -func (fs *FileSystem) savePackageInfo(opts pkgmgmt.InstallOptions) error { +func (fs *FileSystem) savePackageInfo(ctx context.Context, opts pkgmgmt.InstallOptions) error { + log := tracing.LoggerFromContext(ctx) + parentDir, _ := fs.GetPackagesDir() cacheJSONPath := filepath.Join(parentDir, "/", PackageCacheJSON) exists, _ := fs.FileSystem.Exists(cacheJSONPath) if !exists { _, err := fs.FileSystem.Create(cacheJSONPath) if err != nil { - return errors.Wrapf(err, "error creating %s package cache.json", fs.PackageType) + return log.Errorf("error creating %s package cache.json: %w", fs.PackageType, err) } } cacheContentsB, err := fs.FileSystem.ReadFile(cacheJSONPath) if err != nil { - return errors.Wrapf(err, "error reading package %s cache.json", fs.PackageType) + return log.Errorf("error reading package %s cache.json: %w", fs.PackageType, err) } pkgDataJSON := &packages{} if len(cacheContentsB) > 0 { err = json.Unmarshal(cacheContentsB, &pkgDataJSON) if err != nil { - return errors.Wrapf(err, "error unmarshalling from %s package cache.json", fs.PackageType) + return log.Errorf("error unmarshalling from %s package cache.json: %w", fs.PackageType, err) } } //if a package exists, skip. @@ -64,12 +67,12 @@ func (fs *FileSystem) savePackageInfo(opts pkgmgmt.InstallOptions) error { pkgDataJSON.Packages = updatedPkgList updatedPkgInfo, err := json.MarshalIndent(&pkgDataJSON, "", " ") if err != nil { - return errors.Wrapf(err, "error marshalling to %s package cache.json", fs.PackageType) + return log.Errorf("error marshalling to %s package cache.json: %w", fs.PackageType, err) } err = fs.FileSystem.WriteFile(cacheJSONPath, updatedPkgInfo, pkg.FileModeWritable) if err != nil { - return errors.Wrapf(err, "error adding package info to %s cache.json", fs.PackageType) + return log.Errorf("error adding package info to %s cache.json: %w", fs.PackageType, err) } return nil } @@ -84,26 +87,28 @@ type packages struct { Packages []PackageInfo `json:"packages"` } -func (fs *FileSystem) InstallFromURL(opts pkgmgmt.InstallOptions) error { +func (fs *FileSystem) InstallFromURL(ctx context.Context, opts pkgmgmt.InstallOptions) error { clientUrl := opts.GetParsedURL() clientUrl.Path = path.Join(clientUrl.Path, opts.Version, fmt.Sprintf("%s-%s-%s%s", opts.Name, runtime.GOOS, runtime.GOARCH, pkgmgmt.FileExt)) runtimeUrl := opts.GetParsedURL() runtimeUrl.Path = path.Join(runtimeUrl.Path, opts.Version, fmt.Sprintf("%s-linux-amd64", opts.Name)) - return fs.downloadPackage(opts.Name, clientUrl, runtimeUrl) + return fs.downloadPackage(ctx, opts.Name, clientUrl, runtimeUrl) } -func (fs *FileSystem) InstallFromFeedURL(opts pkgmgmt.InstallOptions) error { +func (fs *FileSystem) InstallFromFeedURL(ctx context.Context, opts pkgmgmt.InstallOptions) error { + log := tracing.LoggerFromContext(ctx) + feedUrl := opts.GetParsedFeedURL() tmpDir, err := fs.FileSystem.TempDir("", "porter") if err != nil { - return errors.Wrap(err, "error creating temp directory") + return log.Errorf("error creating temp directory: %w", err) } defer fs.FileSystem.RemoveAll(tmpDir) feedPath := filepath.Join(tmpDir, "atom.xml") - err = fs.downloadFile(feedUrl, feedPath, false) + err = fs.downloadFile(ctx, feedUrl, feedPath, false) if err != nil { return err } @@ -116,23 +121,23 @@ func (fs *FileSystem) InstallFromFeedURL(opts pkgmgmt.InstallOptions) error { result := searchFeed.Search(opts.Name, opts.Version) if result == nil { - return errors.Errorf("the feed at %s does not contain an entry for %s @ %s", opts.FeedURL, opts.Name, opts.Version) + return log.Errorf("the feed at %s does not contain an entry for %s @ %s", opts.FeedURL, opts.Name, opts.Version) } - clientUrl := result.FindDownloadURL(runtime.GOOS, runtime.GOARCH) + clientUrl := result.FindDownloadURL(ctx, runtime.GOOS, runtime.GOARCH) if clientUrl == nil { - return errors.Errorf("%s @ %s did not publish a download for %s/%s", opts.Name, opts.Version, runtime.GOOS, runtime.GOARCH) + return log.Errorf("%s @ %s did not publish a download for %s/%s", opts.Name, opts.Version, runtime.GOOS, runtime.GOARCH) } - runtimeUrl := result.FindDownloadURL("linux", "amd64") + runtimeUrl := result.FindDownloadURL(ctx, "linux", "amd64") if runtimeUrl == nil { - return errors.Errorf("%s @ %s did not publish a download for linux/amd64", opts.Name, opts.Version) + return log.Errorf("%s @ %s did not publish a download for linux/amd64", opts.Name, opts.Version) } - return fs.downloadPackage(opts.Name, *clientUrl, *runtimeUrl) + return fs.downloadPackage(ctx, opts.Name, *clientUrl, *runtimeUrl) } -func (fs *FileSystem) downloadPackage(name string, clientUrl url.URL, runtimeUrl url.URL) error { +func (fs *FileSystem) downloadPackage(ctx context.Context, name string, clientUrl url.URL, runtimeUrl url.URL) error { parentDir, err := fs.GetPackagesDir() if err != nil { return err @@ -140,13 +145,13 @@ func (fs *FileSystem) downloadPackage(name string, clientUrl url.URL, runtimeUrl pkgDir := filepath.Join(parentDir, name) clientPath := fs.BuildClientPath(pkgDir, name) - err = fs.downloadFile(clientUrl, clientPath, true) + err = fs.downloadFile(ctx, clientUrl, clientPath, true) if err != nil { return err } runtimePath := filepath.Join(pkgDir, "runtimes", name+"-runtime") - err = fs.downloadFile(runtimeUrl, runtimePath, true) + err = fs.downloadFile(ctx, runtimeUrl, runtimePath, true) if err != nil { fs.FileSystem.RemoveAll(pkgDir) // If the runtime download fails, cleanup the package so it's not half installed return err @@ -155,22 +160,21 @@ func (fs *FileSystem) downloadPackage(name string, clientUrl url.URL, runtimeUrl return nil } -func (fs *FileSystem) downloadFile(url url.URL, destPath string, executable bool) error { - if fs.Debug { - fmt.Fprintf(fs.Err, "Downloading %s to %s\n", url.String(), destPath) - } +func (fs *FileSystem) downloadFile(ctx context.Context, url url.URL, destPath string, executable bool) error { + log := tracing.LoggerFromContext(ctx) + log.Debugf("Downloading %s to %s\n", url.String(), destPath) req, err := http.NewRequest(http.MethodGet, url.String(), nil) if err != nil { - return errors.Wrapf(err, "error creating web request to %s", url.String()) + return log.Errorf("error creating web request to %s: %w", url.String(), err) } resp, err := http.DefaultClient.Do(req) if err != nil { - return errors.Wrapf(err, "error downloading %s", url.String()) + return log.Errorf("error downloading %s: %w", url.String(), err) } if resp.StatusCode != 200 { - return errors.Errorf("bad status returned when downloading %s (%d) %s", url.String(), resp.StatusCode, resp.Status) + return log.Errorf("bad status returned when downloading %s (%d) %s", url.String(), resp.StatusCode, resp.Status) } defer resp.Body.Close() @@ -178,14 +182,14 @@ func (fs *FileSystem) downloadFile(url url.URL, destPath string, executable bool parentDir := filepath.Dir(destPath) parentDirExists, err := fs.FileSystem.DirExists(parentDir) if err != nil { - return errors.Wrapf(err, "unable to check if directory exists %s", parentDir) + return log.Errorf("unable to check if directory exists %s: %w", parentDir, err) } cleanup := func() {} if !parentDirExists { err = fs.FileSystem.MkdirAll(parentDir, pkg.FileModeDirectory) if err != nil { - errors.Wrapf(err, "unable to create parent directory %s", parentDir) + return log.Errorf("unable to create parent directory %s: %w", parentDir, err) } cleanup = func() { fs.FileSystem.RemoveAll(parentDir) // If we can't download the file, don't leave traces of it @@ -195,7 +199,7 @@ func (fs *FileSystem) downloadFile(url url.URL, destPath string, executable bool destFile, err := fs.FileSystem.Create(destPath) if err != nil { cleanup() - return errors.Wrapf(err, "could not create the file at %s", destPath) + return log.Errorf("could not create the file at %s: %w", destPath, err) } defer destFile.Close() @@ -203,14 +207,14 @@ func (fs *FileSystem) downloadFile(url url.URL, destPath string, executable bool err = fs.FileSystem.Chmod(destPath, pkg.FileModeExecutable) if err != nil { cleanup() - return errors.Wrapf(err, "could not set the file as executable at %s", destPath) + return log.Errorf("could not set the file as executable at %s: %w", destPath, err) } } _, err = io.Copy(destFile, resp.Body) if err != nil { cleanup() - return errors.Wrapf(err, "error writing the file to %s", destPath) + return log.Errorf("error writing the file to %s: %w", destPath, err) } return nil } diff --git a/pkg/pkgmgmt/client/install_test.go b/pkg/pkgmgmt/client/install_test.go index 7992fde10..bba53ce9a 100644 --- a/pkg/pkgmgmt/client/install_test.go +++ b/pkg/pkgmgmt/client/install_test.go @@ -1,6 +1,7 @@ package client import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -38,7 +39,7 @@ func TestFileSystem_InstallFromUrl(t *testing.T) { err := opts.Validate([]string{"mypkg"}) require.NoError(t, err, "Validate failed") - err = p.Install(opts) + err = p.Install(context.Background(), opts) require.NoError(t, err) clientPath := "/home/myuser/.porter/packages/mypkg/mypkg" @@ -87,7 +88,7 @@ func TestFileSystem_InstallFromFeedUrl(t *testing.T) { err = opts.Validate([]string{"helm"}) require.NoError(t, err, "Validate failed") - err = p.Install(opts) + err = p.Install(context.Background(), opts) require.NoError(t, err) clientExists, _ := p.FileSystem.Exists("/home/myuser/.porter/packages/helm/helm") @@ -121,7 +122,7 @@ func TestFileSystem_Install_RollbackMissingRuntime(t *testing.T) { err := opts.Validate([]string{"mypkg"}) require.NoError(t, err, "Validate failed") - err = p.Install(opts) + err = p.Install(context.Background(), opts) require.Error(t, err) assert.Contains(t, err.Error(), "bad status returned when downloading") @@ -148,7 +149,7 @@ func TestFileSystem_Install_PackageInfoSavedWhenNoFileExists(t *testing.T) { cacheExists, _ := p.FileSystem.Exists("/home/myuser/.porter/packages/cache.json") assert.False(t, cacheExists) - err = p.savePackageInfo(opts) + err = p.savePackageInfo(context.Background(), opts) require.NoError(t, err) // cache.json should have been created diff --git a/pkg/pkgmgmt/feed/feed.go b/pkg/pkgmgmt/feed/feed.go index 0ff644fdd..a318d2850 100644 --- a/pkg/pkgmgmt/feed/feed.go +++ b/pkg/pkgmgmt/feed/feed.go @@ -1,11 +1,14 @@ package feed import ( + "context" "fmt" "net/url" "strings" "time" + "get.porter.sh/porter/pkg/tracing" + "get.porter.sh/porter/pkg/portercontext" "github.com/Masterminds/semver/v3" ) @@ -70,7 +73,9 @@ type MixinFileset struct { Files []*MixinFile } -func (f *MixinFileset) FindDownloadURL(os string, arch string) *url.URL { +func (f *MixinFileset) FindDownloadURL(ctx context.Context, os string, arch string) *url.URL { + log := tracing.LoggerFromContext(ctx) + match := fmt.Sprintf("%s-%s-%s", f.Mixin, os, arch) for _, file := range f.Files { if strings.Contains(file.URL.Path, match) { diff --git a/pkg/pkgmgmt/pkgmgmt.go b/pkg/pkgmgmt/pkgmgmt.go index 2f68f81b1..71334f6a0 100644 --- a/pkg/pkgmgmt/pkgmgmt.go +++ b/pkg/pkgmgmt/pkgmgmt.go @@ -1,6 +1,7 @@ package pkgmgmt import ( + "context" "net/url" "os/exec" "path" @@ -13,7 +14,7 @@ type PackageManager interface { List() ([]string, error) GetPackageDir(name string) (string, error) GetMetadata(name string) (PackageMetadata, error) - Install(InstallOptions) error + Install(ctx context.Context, opts InstallOptions) error Uninstall(UninstallOptions) error // Run a command against the installed package. diff --git a/pkg/porter/mixins.go b/pkg/porter/mixins.go index 6c64ac68b..b780d5c33 100644 --- a/pkg/porter/mixins.go +++ b/pkg/porter/mixins.go @@ -2,6 +2,7 @@ package porter import ( "bytes" + "context" "fmt" "io/ioutil" "os" @@ -75,8 +76,8 @@ func (p *Porter) ListMixins() ([]mixin.Metadata, error) { return mixins, nil } -func (p *Porter) InstallMixin(opts mixin.InstallOptions) error { - err := p.Mixins.Install(opts.InstallOptions) +func (p *Porter) InstallMixin(ctx context.Context, opts mixin.InstallOptions) error { + err := p.Mixins.Install(ctx, opts.InstallOptions) if err != nil { return err } diff --git a/pkg/porter/mixins_test.go b/pkg/porter/mixins_test.go index f1285eeca..c14d5798f 100644 --- a/pkg/porter/mixins_test.go +++ b/pkg/porter/mixins_test.go @@ -1,6 +1,7 @@ package porter import ( + "context" "os" "testing" @@ -37,7 +38,7 @@ func TestPorter_InstallMixin(t *testing.T) { opts.Name = "exec" opts.URL = "https://example.com" - err := p.InstallMixin(opts) + err := p.InstallMixin(context.Background(), opts) require.NoError(t, err) diff --git a/pkg/porter/plugins.go b/pkg/porter/plugins.go index 5872f9e4c..3ef511110 100644 --- a/pkg/porter/plugins.go +++ b/pkg/porter/plugins.go @@ -1,6 +1,7 @@ package porter import ( + "context" "fmt" "os" "strings" @@ -150,8 +151,8 @@ func (p *Porter) GetPlugin(name string) (*plugins.Metadata, error) { return plugin, nil } -func (p *Porter) InstallPlugin(opts plugins.InstallOptions) error { - err := p.Plugins.Install(opts.InstallOptions) +func (p *Porter) InstallPlugin(ctx context.Context, opts plugins.InstallOptions) error { + err := p.Plugins.Install(ctx, opts.InstallOptions) if err != nil { return err } diff --git a/pkg/porter/plugins_test.go b/pkg/porter/plugins_test.go index 72841f157..ac6283b08 100644 --- a/pkg/porter/plugins_test.go +++ b/pkg/porter/plugins_test.go @@ -1,6 +1,7 @@ package porter import ( + "context" "testing" "get.porter.sh/porter/pkg/pkgmgmt" @@ -220,7 +221,7 @@ func TestPorter_InstallPlugin(t *testing.T) { err := opts.Validate([]string{"plugin1"}) require.NoError(t, err, "Validate failed") - err = p.InstallPlugin(opts) + err = p.InstallPlugin(context.Background(), opts) require.NoError(t, err, "InstallPlugin failed") wantOutput := "installed plugin1 plugin v1.0 (abc123)\n" From b170646e1e986999f0a99ff02c5ad6adf9a2bab8 Mon Sep 17 00:00:00 2001 From: Carolyn Van Slyck Date: Wed, 6 Apr 2022 13:51:03 -0500 Subject: [PATCH 2/5] Fallback to amd64 on M1 when installing binaries When the package client (used for mixins and plugins) is looking for a compatible binary to download, fallback to downloading amd64 for macos when arm64 is not available. None of the mixins or plugins build to arm64 yet, so by falling back to amd64 the command succeeds and relies on the M1 rosetta capabilties to handle running the other format. Signed-off-by: Carolyn Van Slyck --- pkg/pkgmgmt/client/install.go | 21 ++++++-- pkg/pkgmgmt/client/install_test.go | 86 +++++++++++++++++++----------- pkg/pkgmgmt/feed/feed.go | 7 +++ pkg/pkgmgmt/feed/feed_test.go | 46 ++++++++++++++++ 4 files changed, 126 insertions(+), 34 deletions(-) diff --git a/pkg/pkgmgmt/client/install.go b/pkg/pkgmgmt/client/install.go index a9c0761e6..3a85a7f79 100644 --- a/pkg/pkgmgmt/client/install.go +++ b/pkg/pkgmgmt/client/install.go @@ -88,13 +88,26 @@ type packages struct { } func (fs *FileSystem) InstallFromURL(ctx context.Context, opts pkgmgmt.InstallOptions) error { + return fs.installFromURLFor(ctx, opts, runtime.GOOS, runtime.GOARCH) +} + +func (fs *FileSystem) installFromURLFor(ctx context.Context, opts pkgmgmt.InstallOptions, os string, arch string) error { + log := tracing.LoggerFromContext(ctx) + clientUrl := opts.GetParsedURL() - clientUrl.Path = path.Join(clientUrl.Path, opts.Version, fmt.Sprintf("%s-%s-%s%s", opts.Name, runtime.GOOS, runtime.GOARCH, pkgmgmt.FileExt)) + clientUrl.Path = path.Join(clientUrl.Path, opts.Version, fmt.Sprintf("%s-%s-%s%s", opts.Name, os, arch, pkgmgmt.FileExt)) runtimeUrl := opts.GetParsedURL() runtimeUrl.Path = path.Join(runtimeUrl.Path, opts.Version, fmt.Sprintf("%s-linux-amd64", opts.Name)) - return fs.downloadPackage(ctx, opts.Name, clientUrl, runtimeUrl) + err := fs.downloadPackage(ctx, opts.Name, clientUrl, runtimeUrl) + if err != nil && os == "darwin" && arch == "arm64" { + // Until we have full support for M1 chipsets, rely on rossetta functionality in macos and use the amd64 binary + log.Debugf("%s @ %s did not publish a download for darwin/amd64, falling back to darwin/amd64", opts.Name, opts.Version) + return fs.installFromURLFor(ctx, opts, "darwin", "amd64") + } + + return err } func (fs *FileSystem) InstallFromFeedURL(ctx context.Context, opts pkgmgmt.InstallOptions) error { @@ -174,7 +187,9 @@ func (fs *FileSystem) downloadFile(ctx context.Context, url url.URL, destPath st return log.Errorf("error downloading %s: %w", url.String(), err) } if resp.StatusCode != 200 { - return log.Errorf("bad status returned when downloading %s (%d) %s", url.String(), resp.StatusCode, resp.Status) + err := fmt.Errorf("bad status returned when downloading %s (%d) %s", url.String(), resp.StatusCode, resp.Status) + log.Debugf(err.Error()) // Only debug log this since higher up on the stack we may handle this error + return err } defer resp.Body.Close() diff --git a/pkg/pkgmgmt/client/install_test.go b/pkg/pkgmgmt/client/install_test.go index bba53ce9a..1ed607dba 100644 --- a/pkg/pkgmgmt/client/install_test.go +++ b/pkg/pkgmgmt/client/install_test.go @@ -12,46 +12,70 @@ import ( "strings" "testing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "get.porter.sh/porter/pkg" "get.porter.sh/porter/pkg/config" "get.porter.sh/porter/pkg/pkgmgmt" "get.porter.sh/porter/tests" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestFileSystem_InstallFromUrl(t *testing.T) { - // serve out a fake package - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "#!/usr/bin/env bash\necho i am a random package\n") - })) - defer ts.Close() - - c := config.NewTestConfig(t) - p := NewFileSystem(c.Config, "packages") - - opts := pkgmgmt.InstallOptions{ - PackageType: "mixin", - Version: "latest", - URL: ts.URL, + testcases := []struct { + name string + os string + arch string + responseCode map[string]int + wantError string + }{ + {name: "darwin/arm64 fallback to amd64", os: "darwin", arch: "arm64", responseCode: map[string]int{"arm64": 404}}, + {name: "darwin/arm64 binary exists", os: "darwin", arch: "arm64"}, + {name: "non-darwin arm64 no special handling", os: "myos", arch: "arm64", responseCode: map[string]int{"arm64": 404}, wantError: "404 Not Found"}, } - err := opts.Validate([]string{"mypkg"}) - require.NoError(t, err, "Validate failed") - err = p.Install(context.Background(), opts) - require.NoError(t, err) - - clientPath := "/home/myuser/.porter/packages/mypkg/mypkg" - clientStats, err := p.FileSystem.Stat(clientPath) - require.NoError(t, err) - wantMode := pkg.FileModeExecutable - tests.AssertFilePermissionsEqual(t, clientPath, wantMode, clientStats.Mode()) - - runtimePath := "/home/myuser/.porter/packages/mypkg/runtimes/mypkg-runtime" - runtimeStats, _ := p.FileSystem.Stat(runtimePath) - require.NoError(t, err) - tests.AssertFilePermissionsEqual(t, runtimePath, wantMode, runtimeStats.Mode()) + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + // serve out a fake package + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for term, code := range tc.responseCode { + if strings.Contains(r.RequestURI, term) { + w.WriteHeader(code) + break + } + } + fmt.Fprintf(w, "#!/usr/bin/env bash\necho i am a random package\n") + })) + defer ts.Close() + + c := config.NewTestConfig(t) + p := NewFileSystem(c.Config, "packages") + + opts := pkgmgmt.InstallOptions{ + PackageType: "mixin", + Version: "latest", + URL: ts.URL, + } + err := opts.Validate([]string{"mypkg"}) + require.NoError(t, err, "Validate failed") + + err = p.installFromURLFor(context.Background(), opts, tc.os, tc.arch) + if tc.wantError != "" { + tests.RequireErrorContains(t, err, tc.wantError) + } else { + require.NoError(t, err) + clientPath := "/home/myuser/.porter/packages/mypkg/mypkg" + clientStats, err := p.FileSystem.Stat(clientPath) + require.NoError(t, err) + wantMode := pkg.FileModeExecutable + tests.AssertFilePermissionsEqual(t, clientPath, wantMode, clientStats.Mode()) + + runtimePath := "/home/myuser/.porter/packages/mypkg/runtimes/mypkg-runtime" + runtimeStats, _ := p.FileSystem.Stat(runtimePath) + require.NoError(t, err) + tests.AssertFilePermissionsEqual(t, runtimePath, wantMode, runtimeStats.Mode()) + } + }) + } } func TestFileSystem_InstallFromFeedUrl(t *testing.T) { diff --git a/pkg/pkgmgmt/feed/feed.go b/pkg/pkgmgmt/feed/feed.go index a318d2850..e51c04bab 100644 --- a/pkg/pkgmgmt/feed/feed.go +++ b/pkg/pkgmgmt/feed/feed.go @@ -82,6 +82,13 @@ func (f *MixinFileset) FindDownloadURL(ctx context.Context, os string, arch stri return file.URL } } + + // Until we have full support for M1 chipsets, rely on rossetta functionality in macos and use the amd64 binary + if os == "darwin" && arch == "arm64" { + log.Debugf("%s @ %s did not publish a download for darwin/amd64, falling back to darwin/amd64", f.Mixin, f.Version) + return f.FindDownloadURL(ctx, "darwin", "amd64") + } + return nil } diff --git a/pkg/pkgmgmt/feed/feed_test.go b/pkg/pkgmgmt/feed/feed_test.go index 4bb8b6f5f..6e1f253ea 100644 --- a/pkg/pkgmgmt/feed/feed_test.go +++ b/pkg/pkgmgmt/feed/feed_test.go @@ -1,6 +1,8 @@ package feed import ( + "context" + "net/url" "testing" "get.porter.sh/porter/pkg/portercontext" @@ -71,3 +73,47 @@ func TestMixinFeed_Search_Canary(t *testing.T) { require.NotNil(t, result) assert.Equal(t, "v2-canary", result.Version) } + +func TestMixinFileset_FindDownloadURL(t *testing.T) { + t.Run("darwin/arm64 fallback to amd64", func(t *testing.T) { + link, _ := url.Parse("https://example.com/mymixin-darwin-amd64") + + fs := MixinFileset{ + Mixin: "mymixin", + Files: []*MixinFile{ + {URL: link}, + }, + } + + result := fs.FindDownloadURL(context.Background(), "darwin", "arm64") + assert.Contains(t, result.String(), "amd64", "When an arm64 binary is not available for mac, fallback to using an amd64") + }) + + t.Run("darwin/arm64 binary exists", func(t *testing.T) { + link, _ := url.Parse("https://example.com/mymixin-darwin-arm64") + + fs := MixinFileset{ + Mixin: "mymixin", + Files: []*MixinFile{ + {URL: link}, + }, + } + + result := fs.FindDownloadURL(context.Background(), "darwin", "arm64") + assert.Contains(t, result.String(), "arm64", "When an arm64 binary is available, use it") + }) + + t.Run("non-darwin arm64 no special handling", func(t *testing.T) { + link, _ := url.Parse("https://example.com/mymixin-myos-amd64") + + fs := MixinFileset{ + Mixin: "mymixin", + Files: []*MixinFile{ + {URL: link}, + }, + } + + result := fs.FindDownloadURL(context.Background(), "myos", "arm64") + assert.Nil(t, result) + }) +} From 884a3d70b08d400070dc43732b559b5daa561552 Mon Sep 17 00:00:00 2001 From: Carolyn Van Slyck Date: Wed, 6 Apr 2022 13:53:14 -0500 Subject: [PATCH 3/5] Sync cli docs Signed-off-by: Carolyn Van Slyck --- docs/content/cli/build.md | 2 +- docs/content/cli/bundles_build.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/cli/build.md b/docs/content/cli/build.md index 89cefda06..9ba97abfd 100644 --- a/docs/content/cli/build.md +++ b/docs/content/cli/build.md @@ -34,7 +34,7 @@ porter build [flags] -f, --file porter.yaml Path to the Porter manifest. Defaults to porter.yaml in the current directory. -h, --help help for build --name string Override the bundle name - --no-cache Do not use cache when building the image. + --no-cache Do not use the Docker cache when building the bundle's invocation image. --no-lint Do not run the linter --secret stringArray Secret file to expose to the build (format: id=mysecret,src=/local/secret). May be specified multiple times. --ssh stringArray SSH agent socket or keys to expose to the build (format: default|[=|[,]]). May be specified multiple times. diff --git a/docs/content/cli/bundles_build.md b/docs/content/cli/bundles_build.md index 86f4f3acb..7854d3b35 100644 --- a/docs/content/cli/bundles_build.md +++ b/docs/content/cli/bundles_build.md @@ -34,7 +34,7 @@ porter bundles build [flags] -f, --file porter.yaml Path to the Porter manifest. Defaults to porter.yaml in the current directory. -h, --help help for build --name string Override the bundle name - --no-cache Do not use cache when building the image. + --no-cache Do not use the Docker cache when building the bundle's invocation image. --no-lint Do not run the linter --secret stringArray Secret file to expose to the build (format: id=mysecret,src=/local/secret). May be specified multiple times. --ssh stringArray SSH agent socket or keys to expose to the build (format: default|[=|[,]]). May be specified multiple times. From 8f690f4a0d35b6d83fd7e4e38e8e1b0570e03f30 Mon Sep 17 00:00:00 2001 From: Carolyn Van Slyck Date: Wed, 6 Apr 2022 15:32:29 -0500 Subject: [PATCH 4/5] Use a newer version of magefiles with arm support Signed-off-by: Carolyn Van Slyck --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 99d692f3a..54d54ed3b 100644 --- a/go.mod +++ b/go.mod @@ -18,12 +18,12 @@ replace ( ) require ( - get.porter.sh/magefiles v0.1.2 + get.porter.sh/magefiles v0.1.3 github.com/Masterminds/semver/v3 v3.1.1 github.com/PaesslerAG/jsonpath v0.1.1 github.com/carolynvs/aferox v0.3.0 github.com/carolynvs/datetime-printer v0.2.0 - github.com/carolynvs/magex v0.7.0 + github.com/carolynvs/magex v0.7.2 github.com/cbroglie/mustache v1.0.1 github.com/cnabio/cnab-go v0.23.1 github.com/cnabio/cnab-to-oci v0.3.3 diff --git a/go.sum b/go.sum index 04e6d661a..cc803ccb8 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,8 @@ contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -get.porter.sh/magefiles v0.1.2 h1:9AybljWeUDGKhMwwHYa8wnvtZ1VDPv5i3uHZTwnIysI= -get.porter.sh/magefiles v0.1.2/go.mod h1:wtCIGWp79ARl8Agt1JE4Wchtb0Pn9fea/7LbkOnFbEc= +get.porter.sh/magefiles v0.1.3 h1:91Y7vFDHGmMBbRfHQqEcIlqpp/RfDCMhyHGVusTYlmE= +get.porter.sh/magefiles v0.1.3/go.mod h1:Whw/DSX8dcgn7dUPb6csbY6PtuTy1wujwFR2G6ONf20= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= @@ -276,8 +276,8 @@ github.com/carolynvs/aferox v0.3.0 h1:CMT50zX88amTMbFfFIWSTKRVRaOw6sejUMbbKiCD4z github.com/carolynvs/aferox v0.3.0/go.mod h1:eb7CHGIO33CCZS//xtnblvPZbuuZMv0p1VbhiSwZnH4= github.com/carolynvs/datetime-printer v0.2.0 h1:Td3FU4YGzx0OogCMhCmLBTUTDPQcq0xlgCeMhAKZmMc= github.com/carolynvs/datetime-printer v0.2.0/go.mod h1:p9W8ZUhmQUOVD5kiDuGXwRG65/nTkZWlLylY7s+Qw2k= -github.com/carolynvs/magex v0.7.0 h1:z5PaWogvA/kOHMYueXqlQBobXt32N/a7kZXPvK9V728= -github.com/carolynvs/magex v0.7.0/go.mod h1:vZB3BkRfkd5ZMtkxJkCGbdFyWGoZiuNPKhx6uEQARmY= +github.com/carolynvs/magex v0.7.2 h1:6IpoxUorSDOXtZmd72ExPCzQBxkeUj09JwJfnzf+9y8= +github.com/carolynvs/magex v0.7.2/go.mod h1:vZB3BkRfkd5ZMtkxJkCGbdFyWGoZiuNPKhx6uEQARmY= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A= github.com/cbroglie/mustache v1.0.1 h1:ivMg8MguXq/rrz2eu3tw6g3b16+PQhoTn6EZAhst2mw= From b7e76e8812391eb795486d11fa24445f39416107 Mon Sep 17 00:00:00 2001 From: Carolyn Van Slyck Date: Thu, 7 Apr 2022 09:38:12 -0500 Subject: [PATCH 5/5] Fix log message when falling back to amd64 binaries on darwin Signed-off-by: Carolyn Van Slyck --- pkg/pkgmgmt/feed/feed.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pkgmgmt/feed/feed.go b/pkg/pkgmgmt/feed/feed.go index e51c04bab..e1d8c2c91 100644 --- a/pkg/pkgmgmt/feed/feed.go +++ b/pkg/pkgmgmt/feed/feed.go @@ -85,7 +85,7 @@ func (f *MixinFileset) FindDownloadURL(ctx context.Context, os string, arch stri // Until we have full support for M1 chipsets, rely on rossetta functionality in macos and use the amd64 binary if os == "darwin" && arch == "arm64" { - log.Debugf("%s @ %s did not publish a download for darwin/amd64, falling back to darwin/amd64", f.Mixin, f.Version) + log.Debugf("%s @ %s did not publish a download for darwin/arm64, falling back to darwin/amd64", f.Mixin, f.Version) return f.FindDownloadURL(ctx, "darwin", "amd64") }