Skip to content

Commit

Permalink
fakestorage: make custom client always use the mux transport
Browse files Browse the repository at this point in the history 
For projects that use fake-gcs-server as a library, connecting via TCP
shouldn't be needed (in an ideal world I would make this the default
behavior, but I don't want to introduce a breaking change).

Closes #458.
  • Loading branch information
@fsouza
fsouza committed Mar 21, 2021
1 parent 1eb358f commit 9e93277
Show file tree
Hide file tree
Showing 3 changed files with 134 additions and 65 deletions.
33 changes: 20 additions & 13 deletions fakestorage/object_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1349,20 +1349,27 @@ func TestParseRangeRequest(t *testing.T) {
}

for _, test := range tests {
start, length := test.Start, test.Length
test := test
t.Run("", func(t *testing.T) {
t.Parallel()
start, length := test.Start, test.Length

rng, _ := obj.NewRangeReader(ctx, start, length)
out, _ := ioutil.ReadAll(rng)
rng.Close()
rng, err := obj.NewRangeReader(ctx, start, length)
if err != nil {
t.Fatal(err)
}
out, _ := ioutil.ReadAll(rng)
rng.Close()

if length < 0 {
length = int64(len(in)) - start
}
if n := int64(len(out)); n != length {
t.Fatalf("expected %d bytes, RangeReader returned %d bytes", length, n)
}
if expected := in[start : start+length]; !bytes.Equal(expected, out) {
t.Fatalf("expected %q, RangeReader returned %q", expected, out)
}
if length < 0 {
length = int64(len(in)) - start
}
if n := int64(len(out)); n != length {
t.Fatalf("expected %d bytes, RangeReader returned %d bytes", length, n)
}
if expected := in[start : start+length]; !bytes.Equal(expected, out) {
t.Fatalf("expected %q, RangeReader returned %q", expected, out)
}
})
}
}
46 changes: 22 additions & 24 deletions fakestorage/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ package fakestorage
import (
"compress/gzip"
"context"
"crypto/tls"
"fmt"
"io"
"net"
Expand All @@ -22,6 +21,8 @@ import (
"google.golang.org/api/option"
)

const defaultPublicHost = "storage.googleapis.com"

// Server is the fake server.
//
// It provides a fake implementation of the Google Cloud Storage API.
Expand All @@ -31,6 +32,7 @@ type Server struct {
transport http.RoundTripper
ts *httptest.Server
mux *mux.Router
options Options
externalURL string
publicHost string
}
Expand Down Expand Up @@ -94,7 +96,7 @@ type Options struct {
// NewServerWithOptions creates a new server configured according to the
// provided options.
func NewServerWithOptions(options Options) (*Server, error) {
s, err := newServer(options.InitialObjects, options.StorageRoot, options.ExternalURL, options.PublicHost)
s, err := newServer(options)
if err != nil {
return nil, err
}
Expand All @@ -121,8 +123,8 @@ func NewServerWithOptions(options Options) (*Server, error) {
handler = handlers.LoggingHandler(options.Writer, handler)
}
handler = requestCompressHandler(handler)
s.setTransportToMux(handler)
if options.NoListener {
s.setTransportToMux(handler)
return s, nil
}

Expand All @@ -142,46 +144,36 @@ func NewServerWithOptions(options Options) (*Server, error) {
}
startFunc()

s.setTransportToAddr(s.ts.Listener.Addr().String())
return s, nil
}

func newServer(objects []Object, storageRoot, externalURL, publicHost string) (*Server, error) {
backendObjects := toBackendObjects(objects)
func newServer(options Options) (*Server, error) {
backendObjects := toBackendObjects(options.InitialObjects)
var backendStorage backend.Storage
var err error
if storageRoot != "" {
backendStorage, err = backend.NewStorageFS(backendObjects, storageRoot)
if options.StorageRoot != "" {
backendStorage, err = backend.NewStorageFS(backendObjects, options.StorageRoot)
} else {
backendStorage = backend.NewStorageMemory(backendObjects)
}
if err != nil {
return nil, err
}
publicHost := options.PublicHost
if publicHost == "" {
publicHost = "storage.googleapis.com"
publicHost = defaultPublicHost
}
s := Server{
backend: backendStorage,
uploads: sync.Map{},
externalURL: externalURL,
externalURL: options.ExternalURL,
publicHost: publicHost,
options: options,
}
s.buildMuxer()
return &s, nil
}

func (s *Server) setTransportToAddr(addr string) {
// #nosec
tlsConfig := tls.Config{InsecureSkipVerify: true}
s.transport = &http.Transport{
TLSClientConfig: &tlsConfig,
DialTLS: func(string, string) (net.Conn, error) {
return tls.Dial("tcp", addr, &tlsConfig)
},
}
}

func (s *Server) setTransportToMux(handler http.Handler) {
s.transport = &muxTransport{handler: handler}
}
Expand Down Expand Up @@ -247,7 +239,14 @@ func (s *Server) URL() string {

// PublicURL returns the server's public download URL.
func (s *Server) PublicURL() string {
return fmt.Sprintf("https://%s", s.publicHost)
return fmt.Sprintf("%s://%s", s.scheme(), s.publicHost)
}

func (s *Server) scheme() string {
if s.options.Scheme == "http" {
return "http"
}
return "https"
}

// HTTPClient returns an HTTP client configured to talk to the server.
Expand All @@ -257,8 +256,7 @@ func (s *Server) HTTPClient() *http.Client {

// Client returns a GCS client configured to talk to the server.
func (s *Server) Client() *storage.Client {
opt := option.WithHTTPClient(s.HTTPClient())
client, _ := storage.NewClient(context.Background(), opt)
client, _ := storage.NewClient(context.Background(), option.WithHTTPClient(s.HTTPClient()))
return client
}

Expand Down
120 changes: 92 additions & 28 deletions fakestorage/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,60 @@ func TestNewServerLogging(t *testing.T) {
}
}

func TestPublicURL(t *testing.T) {
t.Parallel()
var tests = []struct {
name string
options Options
expected string
}{
{
name: "https scheme",
options: Options{Scheme: "https", NoListener: true},
expected: "https://storage.googleapis.com",
},
{
name: "http scheme",
options: Options{Scheme: "http", NoListener: true},
expected: "http://storage.googleapis.com",
},
{
name: "no scheme",
options: Options{NoListener: true},
expected: "https://storage.googleapis.com",
},
{
name: "https scheme - custom public host",
options: Options{Scheme: "https", NoListener: true, PublicHost: "localhost:8080"},
expected: "https://localhost:8080",
},
{
name: "no scheme - custom public host",
options: Options{NoListener: true, PublicHost: "localhost:8080"},
expected: "https://localhost:8080",
},
{
name: "http scheme - custom public host",
options: Options{Scheme: "http", NoListener: true, PublicHost: "localhost:8080"},
expected: "http://localhost:8080",
},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
t.Parallel()
server, err := NewServerWithOptions(test.options)
if err != nil {
t.Fatal(err)
}
defer server.Stop()
if publicURL := server.PublicURL(); publicURL != test.expected {
t.Errorf("wrong public url returned\nwant %q\ngot %q", test.expected, publicURL)
}
})
}
}

func TestDownloadObject(t *testing.T) {
objs := []Object{
{BucketName: "some-bucket", Name: "files/txt/text-01.txt", Content: []byte("something")},
Expand All @@ -101,48 +155,46 @@ func testDownloadObject(t *testing.T, server *Server) {
{
"GET: bucket in the path",
http.MethodGet,
"https://storage.googleapis.com/some-bucket/files/txt/text-01.txt",
"://storage.googleapis.com/some-bucket/files/txt/text-01.txt",
map[string]string{"accept-ranges": "bytes", "content-length": "9"},
"something",
},
{
"GET: bucket in the host",
http.MethodGet,
"https://other-bucket.storage.googleapis.com/static/css/website.css",
"://other-bucket.storage.googleapis.com/static/css/website.css",
map[string]string{"accept-ranges": "bytes", "content-length": "21"},
"body {display: none;}",
},
{
"GET: using storage api",
http.MethodGet,
"https://storage.googleapis.com/storage/v1/b/some-bucket/o/files/txt/text-01.txt?alt=media",
"://storage.googleapis.com/storage/v1/b/some-bucket/o/files/txt/text-01.txt?alt=media",
map[string]string{"accept-ranges": "bytes", "content-length": "9"},
"something",
},
{
"HEAD: bucket in the path",
http.MethodHead,
"https://storage.googleapis.com/some-bucket/files/txt/text-01.txt",
"://storage.googleapis.com/some-bucket/files/txt/text-01.txt",
map[string]string{"accept-ranges": "bytes", "content-length": "9"},
"",
},
{
"HEAD: bucket in the host",
http.MethodHead,
"https://other-bucket.storage.googleapis.com/static/css/website.css",
"://other-bucket.storage.googleapis.com/static/css/website.css",
map[string]string{"accept-ranges": "bytes", "content-length": "21"},
"",
},
}
expected := "https://storage.googleapis.com"
if server.PublicURL() != expected {
t.Fatalf("Expected PublicURL \"%s\", is \"%s\".", expected, server.PublicURL())
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
t.Parallel()
client := server.HTTPClient()
req, err := http.NewRequest(test.method, test.url, nil)
url := server.scheme() + test.url
req, err := http.NewRequest(test.method, url, nil)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -186,7 +238,7 @@ func testDownloadObjectRange(t *testing.T, server *Server) {
test := test
t.Run(test.name, func(t *testing.T) {
client := server.HTTPClient()
req, err := http.NewRequest("GET", "https://storage.googleapis.com/some-bucket/files/txt/text-01.txt", nil)
req, err := http.NewRequest("GET", server.scheme()+"://storage.googleapis.com/some-bucket/files/txt/text-01.txt", nil)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -372,21 +424,33 @@ func TestCORSRequests(t *testing.T) {
}

func runServersTest(t *testing.T, objs []Object, fn func(*testing.T, *Server)) {
t.Run("tcp listener", func(t *testing.T) {
t.Parallel()
tcpServer, err := NewServerWithOptions(Options{NoListener: false, InitialObjects: objs})
if err != nil {
t.Fatal(err)
}
defer tcpServer.Stop()
fn(t, tcpServer)
})
t.Run("no listener", func(t *testing.T) {
t.Parallel()
noListenerServer, err := NewServerWithOptions(Options{NoListener: true, InitialObjects: objs})
if err != nil {
t.Fatal(err)
}
fn(t, noListenerServer)
})
var testScenarios = []struct {
name string
options Options
}{
{
name: "https listener",
options: Options{NoListener: false, InitialObjects: objs},
},
{
name: "http listener",
options: Options{Scheme: "http", NoListener: false, InitialObjects: objs},
},
{
name: "no listener",
options: Options{NoListener: true, InitialObjects: objs},
},
}
for _, test := range testScenarios {
test := test
t.Run(test.name, func(t *testing.T) {
t.Parallel()
server, err := NewServerWithOptions(test.options)
if err != nil {
t.Fatal(err)
}
defer server.Stop()
fn(t, server)
})
}
}

0 comments on commit 9e93277

Please sign in to comment.