support download from http link (#7)

* support download from http link

Signed-off-by: runzexia <[email protected]>

* fix comment

Signed-off-by: runzexia <[email protected]>

Makefile

@@ -3,7 +3,9 @@ build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o _output/cmd/builder github.com/kubesphere/s2irun/cmd
 run: 
 	S2I_CONFIG_PATH=test/config.json go run ./cmd/main.go
+run-b2i:
+	S2I_CONFIG_PATH=test/b2iconfig.json go run ./cmd/main.go
 image:
 	docker build . -t ${IMG}
 push: image
-	docker push ${IMG}
+	docker push ${IMG}

pkg/api/describe/describer.go

@@ -96,11 +96,11 @@ func Config(client docker.Client, config *api.Config) string {
 
 func describeBuilderImage(client docker.Client, config *api.Config, out io.Writer) {
 	c := &api.Config{
-		DockerConfig:       config.DockerConfig,
-		PullAuthentication: config.PullAuthentication,
-		BuilderImage:       config.BuilderImage,
-		BuilderPullPolicy:  config.BuilderPullPolicy,
-		Tag:                config.Tag,
+		DockerConfig:              config.DockerConfig,
+		PullAuthentication:        config.PullAuthentication,
+		BuilderImage:              config.BuilderImage,
+		BuilderPullPolicy:         config.BuilderPullPolicy,
+		Tag:                       config.Tag,
 		IncrementalAuthentication: config.IncrementalAuthentication,
 	}
 	dkr := docker.New(client, c.PullAuthentication, c.PushAuthentication)

pkg/api/types.go

@@ -261,6 +261,10 @@ type Config struct {
 	// SourceURL is  url of the codes such as https://github.com/a/b.git
 	SourceURL string `json:"sourceURL,omitempty"`
 
+	// IsBinaryURL explain the type of SourceURL.
+	// If it is IsBinaryURL, it will download the file directly without using git.
+	IsBinaryURL bool `json:"isBinaryURL,omitempty"`
+
 	// The RevisionId is a branch name or a SHA-1 hash of every important thing about the commit
 	RevisionId string `json:"revisionId,omitempty"`
 }

pkg/build/config.go

@@ -43,7 +43,7 @@ func GenerateConfigFromLabels(config *api.Config, metadata *docker.PullResult) e
 	}
 
 	if repo, ok := labels[constants.BuildSourceLocationLabel]; ok {
-		source, err := git.Parse(repo)
+		source, err := git.Parse(repo, false)
 		if err != nil {
 			return fmt.Errorf("couldn't parse label %q value %s: %v", constants.BuildSourceLocationLabel, repo, err)
 		}

pkg/build/strategies/onbuild/onbuild_test.go

@@ -7,13 +7,13 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/docker/docker/builder/dockerfile"
 	"github.com/kubesphere/s2irun/pkg/api"
 	"github.com/kubesphere/s2irun/pkg/docker"
 	"github.com/kubesphere/s2irun/pkg/scm/git"
 	"github.com/kubesphere/s2irun/pkg/test"
 	testfs "github.com/kubesphere/s2irun/pkg/test/fs"
 	"github.com/kubesphere/s2irun/pkg/utils/fs"
-	"github.com/docker/docker/builder/dockerfile"
 )
 
 type fakeSourceHandler struct{}

pkg/run/s2i.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"os"
 	"github.com/kubesphere/s2irun/pkg/api"
 	"github.com/kubesphere/s2irun/pkg/api/describe"
 	"github.com/kubesphere/s2irun/pkg/api/validation"
@@ -13,6 +12,7 @@ import (
 	s2ierr "github.com/kubesphere/s2irun/pkg/errors"
 	"github.com/kubesphere/s2irun/pkg/scm/git"
 	utilglog "github.com/kubesphere/s2irun/pkg/utils/glog"
+	"os"
 )
 
 const (
@@ -107,7 +107,7 @@ func App() int {
 		glog.Errorf("There are some errors in config file, please check the error:\n%v", err)
 		return 1
 	}
-	apiConfig.Source, err = git.Parse(apiConfig.SourceURL)
+	apiConfig.Source, err = git.Parse(apiConfig.SourceURL, apiConfig.IsBinaryURL)
 	if err != nil {
 		glog.Errorf("SourceURL is illegal, please check the error:\n%v", err)
 		return 1

pkg/scm/downloaders/binary/download.go

@@ -0,0 +1,82 @@
+package binary
+
+import (
+	"github.com/kubesphere/s2irun/pkg/api"
+	"github.com/kubesphere/s2irun/pkg/api/constants"
+	"github.com/kubesphere/s2irun/pkg/scm/git"
+	"github.com/kubesphere/s2irun/pkg/utils/bytefmt"
+	"github.com/kubesphere/s2irun/pkg/utils/fs"
+	utilglog "github.com/kubesphere/s2irun/pkg/utils/glog"
+	"io"
+	"net/http"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+var glog = utilglog.StderrLog
+
+// File represents a simplest possible Downloader implementation where the
+// sources are just copied from local directory.
+type File struct {
+	fs.FileSystem
+}
+
+// Download download sources from a http link into the working directory.
+// Caller guarantees that config.Source.IsLocal() is true.
+func (f *File) Download(config *api.Config) (*git.SourceInfo, error) {
+	_, filename := filepath.Split(config.Source.String())
+	config.WorkingSourceDir = filepath.Join(config.WorkingDir, constants.Source)
+	binaryPath := filepath.Join(config.WorkingSourceDir, filename)
+	glog.V(0).Infof("Start Download Binary %s", filename)
+	resp, err := http.Get(config.Source.String())
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	out, err := f.Create(binaryPath)
+	if err != nil {
+		return nil, err
+	}
+
+	defer out.Close()
+
+	strsize := resp.Header.Get("Content-Length")
+	size, _ := strconv.ParseUint(strsize, 10, 64)
+
+	counter := &WriteCounter{Size: size}
+	_, err = io.Copy(out, io.TeeReader(resp.Body, counter))
+	glog.V(0).Infof("Finish Download Binary %s", filename)
+	return &git.SourceInfo{
+		Location:   config.Source.String(),
+		ContextDir: config.ContextDir,
+	}, nil
+}
+
+// write cycle.
+type WriteCounter struct {
+	Total uint64
+	Size  uint64
+}
+
+func (wc *WriteCounter) Write(p []byte) (int, error) {
+	n := len(p)
+	wc.Total += uint64(n)
+	wc.PrintProgress()
+	return n, nil
+}
+
+func (wc WriteCounter) PrintProgress() {
+	// Clear the line by using a character return to go back to the start and remove
+	// the remaining characters by filling it with spaces
+	glog.V(0).Infof("\r%s", strings.Repeat(" ", 35))
+
+	// Return again and print current status of download
+	// We use the humanize package to print the bytes in a meaningful way (e.g. 10 MB)
+	if wc.Size > 0 {
+		glog.V(0).Infof("\rDownloading... %s//%s", bytefmt.ByteSize(wc.Total), bytefmt.ByteSize(wc.Size))
+	} else {
+		glog.V(0).Infof("\rDownloading... %s complete", bytefmt.ByteSize(wc.Total))
+	}
+
+}

pkg/scm/downloaders/binary/download_test.go

@@ -0,0 +1,28 @@
+package binary
+
+import (
+	"github.com/kubesphere/s2irun/pkg/api"
+	"github.com/kubesphere/s2irun/pkg/scm/git"
+	testfs "github.com/kubesphere/s2irun/pkg/test/fs"
+	"testing"
+)
+
+func TestDownload(t *testing.T) {
+	fs := &testfs.FakeFileSystem{}
+	f := &File{fs}
+
+	config := &api.Config{
+		Source:      git.MustParse("https://kubesphere.io/etcd-operator.svg"),
+		IsBinaryURL: true,
+	}
+	info, err := f.Download(config)
+	if err != nil {
+		t.Errorf("Unexpected error %v", err)
+	}
+	if fs.CreateFile != "upload/src/etcd-operator.svg" {
+		t.Errorf("Unexpected fs.CreateFile %s", fs.CreateFile)
+	}
+	if info.Location != config.Source.String() || info.ContextDir != config.ContextDir {
+		t.Errorf("Unexpected info")
+	}
+}

pkg/scm/git/url.go

@@ -42,6 +42,8 @@ const (
 	URLTypeSCP
 	// URLTypeLocal is the local type (see above)
 	URLTypeLocal
+	// URLTypeBinary is the URL to download file
+	URLTypeBinary
 )
 
 // String returns a string representation of the URLType
@@ -53,6 +55,8 @@ func (t URLType) String() string {
 		return "URLTypeSCP"
 	case URLTypeLocal:
 		return "URLTypeLocal"
+	case URLTypeBinary:
+		return "URLTypeBinary"
 	}
 	panic("unknown URLType")
 }
@@ -84,7 +88,7 @@ func splitOnByte(s string, c byte) (string, string) {
 }
 
 // Parse parses a "Git URL"
-func Parse(rawurl string) (*URL, error) {
+func Parse(rawurl string, isBinaryURL bool) (*URL, error) {
 	if urlSchemeRegexp.MatchString(rawurl) &&
 		(runtime.GOOS != "windows" || !dosDriveRegexp.MatchString(rawurl)) {
 		u, err := url.Parse(rawurl)
@@ -99,11 +103,17 @@ func Parse(rawurl string) (*URL, error) {
 				return nil, fmt.Errorf("file url %q has non-absolute path %q", rawurl, u.Path)
 			}
 		}
-
-		return &URL{
-			URL:  *u,
-			Type: URLTypeURL,
-		}, nil
+		if isBinaryURL {
+			return &URL{
+				URL:  *u,
+				Type: URLTypeBinary,
+			}, nil
+		} else {
+			return &URL{
+				URL:  *u,
+				Type: URLTypeURL,
+			}, nil
+		}
 	}
 
 	s, fragment := splitOnByte(rawurl, '#')
@@ -136,7 +146,7 @@ func Parse(rawurl string) (*URL, error) {
 
 // MustParse parses a "Git URL" and panics on failure
 func MustParse(rawurl string) *URL {
-	u, err := Parse(rawurl)
+	u, err := Parse(rawurl, false)
 	if err != nil {
 		panic(err)
 	}
@@ -149,6 +159,8 @@ func (u URL) String() string {
 	switch u.Type {
 	case URLTypeURL:
 		return u.URL.String()
+	case URLTypeBinary:
+		return u.URL.String()
 	case URLTypeSCP:
 		if u.URL.User != nil {
 			s = u.URL.User.Username() + "@"

pkg/scm/git/url_test.go

@@ -209,7 +209,7 @@ func TestParse(t *testing.T) {
 	)
 
 	for _, test := range tests {
-		parsedURL, err := Parse(test.rawurl)
+		parsedURL, err := Parse(test.rawurl, false)
 		if test.expectedError != (err != nil) {
 			t.Errorf("%s: Parse() returned err: %v", test.rawurl, err)
 		}

pkg/scm/scm.go

@@ -3,6 +3,7 @@ package scm
 import (
 	"github.com/kubesphere/s2irun/pkg/build"
 	"github.com/kubesphere/s2irun/pkg/errors"
+	"github.com/kubesphere/s2irun/pkg/scm/downloaders/binary"
 	"github.com/kubesphere/s2irun/pkg/scm/downloaders/empty"
 	"github.com/kubesphere/s2irun/pkg/scm/downloaders/file"
 	gitdownloader "github.com/kubesphere/s2irun/pkg/scm/downloaders/git"
@@ -22,7 +23,9 @@ func DownloaderForSource(fs fs.FileSystem, s *git.URL, forceCopy bool) (build.Do
 	if s == nil {
 		return &empty.Noop{}, nil
 	}
-
+	if s.Type == git.URLTypeBinary {
+		return &binary.File{FileSystem: fs}, nil
+	}
 	if s.IsLocal() {
 		if forceCopy {
 			return &file.File{FileSystem: fs}, nil