diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 82876ead5c..1b6419d6c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -147,6 +147,12 @@ jobs: - lang: curl docker-image: alpine entrypoint: /bin/sh + - lang: curl-both-scheme-default-ports + docker-image: alpine + entrypoint: /bin/sh + - lang: curl-both-scheme-non-default-ports + docker-image: alpine + entrypoint: /bin/sh name: test-${{ matrix.lang }}-example runs-on: ubuntu-latest diff --git a/README.md b/README.md index 3b9e203543..28bda0d328 100644 --- a/README.md +++ b/README.md @@ -7,18 +7,18 @@ fake-gcs-server provides an emulator for Google Cloud Storage API. It can be used as a library in Go projects and/or as a standalone binary/Docker image. The library is available inside the package -[``github.com/fsouza/fake-gcs-server/fakestorage``](https://pkg.go.dev/github.com/fsouza/fake-gcs-server/fakestorage?tab=doc) +[`github.com/fsouza/fake-gcs-server/fakestorage`](https://pkg.go.dev/github.com/fsouza/fake-gcs-server/fakestorage?tab=doc) and can be used from within test suites in Go package. The emulator is available as a binary that can be built manually, downloaded from the [releases page](https://github.com/fsouza/fake-gcs-server/releases) or pulled from Docker -Hub ([``docker pull -fsouza/fake-gcs-server``](https://hub.docker.com/r/fsouza/fake-gcs-server)). +Hub ([`docker pull +fsouza/fake-gcs-server`](https://hub.docker.com/r/fsouza/fake-gcs-server)). ## Using the emulator in Docker You can stub/mock Google Cloud Storage as a standalone server (like the datastore/pubsub emulators) which is ideal for integration tests and/or tests in other languages you may want to run the -``fake-gcs-server`` inside a Docker container: +`fake-gcs-server` inside a Docker container: ```shell docker run -d --name fake-gcs-server -p 4443:4443 fsouza/fake-gcs-server @@ -26,14 +26,14 @@ docker run -d --name fake-gcs-server -p 4443:4443 fsouza/fake-gcs-server ### Preload data -In case you want to preload some data in ``fake-gcs-server`` just mount a -folder in the container at ``/data``: +In case you want to preload some data in `fake-gcs-server` just mount a +folder in the container at `/data`: ```shell docker run -d --name fake-gcs-server -p 4443:4443 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server ``` -Where the content of ``${PWD}/examples/data`` is something like: +Where the content of `${PWD}/examples/data` is something like: ``` . @@ -51,13 +51,14 @@ curl --insecure https://0.0.0.0:4443/storage/v1/b/sample-bucket/o {"kind":"storage#objects","items":[{"kind":"storage#object","name":"some_file.txt","id":"sample-bucket/some_file.txt","bucket":"sample-bucket","size":"33"}],"prefixes":[]} ``` -This will result in one bucket called ``sample-bucket`` containing one object called ``some_file.txt``. +This will result in one bucket called `sample-bucket` containing one object called `some_file.txt`. ### Running with HTTP fake-gcs-server defaults to HTTPS, but it can also be used with HTTP. The flag -`-scheme` can be used to specify the protocol. For example, the previous -example could be changed to pass `-scheme http`: +`-scheme` can be used to specify the protocol. +The binding port will be `-port` (defaults to `4443`). +For example, the previous example could be changed to pass `-scheme http`: ```shell docker run -d --name fake-gcs-server -p 4443:4443 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server -scheme http @@ -74,6 +75,16 @@ curl http://0.0.0.0:4443/storage/v1/b/sample-bucket/o {"kind":"storage#objects","items":[{"kind":"storage#object","name":"some_file.txt","id":"sample-bucket/some_file.txt","bucket":"sample-bucket","size":"33"}],"prefixes":[]} ``` +### Running with both HTTPS and HTTP + +To start both HTTPS and HTTP servers, pass `-scheme both`. +HTTPS will bind to `-port` (defaults to `4443`) and HTTP will bind to `-port-http` (defaults to `8000`). +For example, the previous example could be changed to pass `-scheme both`: + +```shell +docker run -d --name fake-gcs-server -p 4443:4443 -p 8000:8000 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server -scheme both +``` + ### Using with signed URLs It is possible to use fake-gcs-server with signed URLs, although with a few caveats: @@ -97,11 +108,11 @@ docker run --rm fsouza/fake-gcs-server -help ## Client library examples For examples using SDK from multiple languages, check out the -[``examples``](/examples/) directory. +[`examples`](/examples/) directory. ### Building the image locally -You may use ``docker build`` to build the image locally instead of pulling it +You may use `docker build` to build the image locally instead of pulling it from Docker Hub: ```shell diff --git a/ci/run-curl-both-scheme-default-ports-example.sh b/ci/run-curl-both-scheme-default-ports-example.sh new file mode 100644 index 0000000000..b9ac4d038a --- /dev/null +++ b/ci/run-curl-both-scheme-default-ports-example.sh @@ -0,0 +1,11 @@ +# Copyright 2023 Francisco Souza. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -e + +./fake-gcs-server -backend memory -scheme both -data ${PWD}/examples/data & + +apk add --update curl +curl --silent --fail --insecure https://0.0.0.0:4443/storage/v1/b +curl --silent --fail --insecure http://0.0.0.0:8000/storage/v1/b diff --git a/ci/run-curl-both-scheme-non-default-ports-example.sh b/ci/run-curl-both-scheme-non-default-ports-example.sh new file mode 100644 index 0000000000..4b53bafffe --- /dev/null +++ b/ci/run-curl-both-scheme-non-default-ports-example.sh @@ -0,0 +1,11 @@ +# Copyright 2023 Francisco Souza. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -e + +./fake-gcs-server -backend memory -port 5553 -port-http 9000 -scheme both -data ${PWD}/examples/data & + +apk add --update curl +curl --silent --fail --insecure https://0.0.0.0:5553/storage/v1/b +curl --silent --fail --insecure http://0.0.0.0:9000/storage/v1/b diff --git a/internal/config/config.go b/internal/config/config.go index 3302b1778e..24b2c41336 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,6 +24,13 @@ const ( eventDelete = "delete" eventMetadataUpdate = "metadataUpdate" eventArchive = "archive" + defaultHTTPSPort = 4443 + defaultHTTPPort = 8000 + schemeHTTPS = "https" + schemeHTTP = "http" + schemeBoth = "both" + flagPort = "port" + flagPortHTTP = "port-http" ) type Config struct { @@ -31,6 +38,7 @@ type Config struct { Seed string Host string Port uint + PortHTTP uint CertificateLocation string PrivateKeyLocation string @@ -65,11 +73,12 @@ func Load(args []string) (Config, error) { fs.StringVar(&cfg.fsRoot, "filesystem-root", "/storage", "filesystem root (required for the filesystem backend). folder will be created if it doesn't exist") fs.StringVar(&cfg.publicHost, "public-host", "storage.googleapis.com", "Optional URL for public host") fs.StringVar(&cfg.externalURL, "external-url", "", "optional external URL, returned in the Location header for uploads. Defaults to the address where the server is running") - fs.StringVar(&cfg.Scheme, "scheme", "https", "using http or https") + fs.StringVar(&cfg.Scheme, "scheme", schemeHTTPS, "using 'http' or 'https' or 'both'") fs.StringVar(&cfg.Host, "host", "0.0.0.0", "host to bind to") fs.StringVar(&cfg.Seed, "data", "", "where to load data from (provided that the directory exists)") fs.StringVar(&allowedCORSHeaders, "cors-headers", "", "comma separated list of headers to add to the CORS allowlist") - fs.UintVar(&cfg.Port, "port", 4443, "port to bind to") + fs.UintVar(&cfg.Port, flagPort, 0, "port to which https (default 4443) or http (default 8000) will be bound, based on the specified scheme. If the scheme is 'both', then bind to https") + fs.UintVar(&cfg.PortHTTP, flagPortHTTP, 0, "used only when scheme is 'both' as the port to bind http to (default 8000)") fs.StringVar(&cfg.event.pubsubProjectID, "event.pubsub-project-id", "", "project ID containing the pubsub topic") fs.StringVar(&cfg.event.pubsubTopic, "event.pubsub-topic", "", "pubsub topic name to publish events on") fs.StringVar(&cfg.event.bucket, "event.bucket", "", "if not empty, only objects in this bucket will generate trigger events") @@ -85,6 +94,27 @@ func Load(args []string) (Config, error) { return cfg, err } + // Create a map to store the flags and their values + setFlags := make(map[string]interface{}) + + // Check if a flag was used using Visit + fs.Visit(func(f *flag.Flag) { + setFlags[f.Name] = f.Value + }) + + // setting default values, if not provided, for port and http ports based on scheme value + if _, ok := setFlags[flagPort]; !ok { + if cfg.Scheme == schemeHTTPS || cfg.Scheme == schemeBoth { + cfg.Port = defaultHTTPSPort + } else if cfg.Scheme == schemeHTTP { + cfg.Port = defaultHTTPPort + } + } + + if _, ok := setFlags[flagPortHTTP]; !ok && cfg.Scheme == schemeBoth { + cfg.PortHTTP = defaultHTTPPort + } + cfg.LogLevel, err = logrus.ParseLevel(logLevel) if err != nil { return cfg, err @@ -98,7 +128,12 @@ func Load(args []string) (Config, error) { } if cfg.externalURL == "" { - cfg.externalURL = fmt.Sprintf("%s://%s:%d", cfg.Scheme, cfg.Host, cfg.Port) + if cfg.Scheme != "both" { + cfg.externalURL = fmt.Sprintf("%s://%s:%d", cfg.Scheme, cfg.Host, cfg.Port) + } else { + // for scheme 'both' taking externalURL as HTTPs by default + cfg.externalURL = fmt.Sprintf("%s://%s:%d", schemeHTTPS, cfg.Host, cfg.Port) + } } return cfg, cfg.validate() @@ -111,12 +146,15 @@ func (c *Config) validate() error { if c.backend == filesystemBackend && c.fsRoot == "" { return fmt.Errorf("backend %q requires the filesystem-root to be defined", c.backend) } - if c.Scheme != "http" && c.Scheme != "https" { - return fmt.Errorf(`invalid scheme %s, must be either "http"" or "https"`, c.Scheme) + if c.Scheme != schemeHTTP && c.Scheme != schemeHTTPS && c.Scheme != schemeBoth { + return fmt.Errorf(`invalid scheme %s, must be either "%s", "%s" or "%s"`, c.Scheme, schemeHTTP, schemeHTTPS, schemeBoth) } if c.Port > math.MaxUint16 { return fmt.Errorf("port %d is too high, maximum value is %d", c.Port, math.MaxUint16) } + if c.PortHTTP > math.MaxUint16 { + return fmt.Errorf("port-http %d is too high, maximum value is %d", c.PortHTTP, math.MaxUint16) + } return c.event.validate() } @@ -148,7 +186,7 @@ func (c *EventConfig) validate() error { return nil } -func (c *Config) ToFakeGcsOptions() fakestorage.Options { +func (c *Config) ToFakeGcsOptions(scheme string) fakestorage.Options { storageRoot := c.fsRoot if c.backend == memoryBackend { storageRoot = "" @@ -173,14 +211,18 @@ func (c *Config) ToFakeGcsOptions() fakestorage.Options { } } } + port := c.Port + if c.Scheme == schemeBoth && scheme == schemeHTTP { + port = c.PortHTTP // this cli flag, for port http, is relevant only when scheme is both + } logger := logrus.New() logger.SetLevel(c.LogLevel) opts := fakestorage.Options{ StorageRoot: storageRoot, Seed: c.Seed, - Scheme: c.Scheme, + Scheme: scheme, Host: c.Host, - Port: uint16(c.Port), + Port: uint16(port), PublicHost: c.publicHost, ExternalURL: c.externalURL, AllowedCORSHeaders: c.allowedCORSHeaders, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9e33018ff4..ca391d5570 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -32,8 +32,9 @@ func TestLoadConfig(t *testing.T) { "-cors-headers", "X-Goog-Meta-Uploader", "-host", "127.0.0.1", "-port", "443", + "-port-http", "80", "-data", "/var/gcs", - "-scheme", "http", + "-scheme", "both", "-event.pubsub-project-id", "test-project", "-event.pubsub-topic", "gcs-events", "-event.object-prefix", "uploads/", @@ -50,7 +51,8 @@ func TestLoadConfig(t *testing.T) { allowedCORSHeaders: []string{"X-Goog-Meta-Uploader"}, Host: "127.0.0.1", Port: 443, - Scheme: "http", + PortHTTP: 80, + Scheme: "both", event: EventConfig{ pubsubProjectID: "test-project", pubsubTopic: "gcs-events", @@ -72,6 +74,77 @@ func TestLoadConfig(t *testing.T) { allowedCORSHeaders: nil, Host: "0.0.0.0", Port: 4443, + PortHTTP: 0, + Scheme: "https", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "https scheme and default port", + args: []string{ + "-scheme", "https", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:4443", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 4443, + PortHTTP: 0, + Scheme: "https", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "http scheme and default port", + args: []string{ + "-scheme", "http", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "http://0.0.0.0:8000", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 8000, + PortHTTP: 0, + Scheme: "http", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "https scheme and non-default port", + args: []string{ + "-port", "5553", + "-scheme", "https", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:5553", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 5553, + PortHTTP: 0, Scheme: "https", event: EventConfig{ list: []string{"finalize"}, @@ -80,16 +153,151 @@ func TestLoadConfig(t *testing.T) { LogLevel: logrus.InfoLevel, }, }, + { + name: "http scheme and non-default port", + args: []string{ + "-port", "9000", + "-scheme", "http", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "http://0.0.0.0:9000", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 9000, + PortHTTP: 0, + Scheme: "http", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "'both' scheme and default ports", + args: []string{ + "-scheme", "both", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:4443", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 4443, + PortHTTP: 8000, + Scheme: "both", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "'both' scheme with non-default https port and default http", + args: []string{ + "-port", "5553", + "-scheme", "both", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:5553", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 5553, + PortHTTP: 8000, + Scheme: "both", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "'both' scheme with default https port and non-default http", + args: []string{ + "-port-http", "9000", + "-scheme", "both", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:4443", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 4443, + PortHTTP: 9000, + Scheme: "both", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "'both' scheme with non-default ports", + args: []string{ + "-port", "5553", + "-port-http", "9000", + "-scheme", "both", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:5553", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 5553, + PortHTTP: 9000, + Scheme: "both", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, { name: "invalid port value type", args: []string{"-port", "not-a-number"}, expectErr: true, }, + { + name: "invalid port-http value type", + args: []string{"-port-http", "not-a-number"}, + expectErr: true, + }, { name: "invalid port value", args: []string{"-port", "65536"}, expectErr: true, }, + { + name: "invalid port-http value", + args: []string{"-port-http", "65536"}, + expectErr: true, + }, + { + name: "invalid scheme value", + args: []string{"-scheme", "wrong-scheme-value"}, + expectErr: true, + }, { name: "invalid backend", args: []string{"-backend", "in-memory"}, @@ -154,6 +362,7 @@ func TestToFakeGcsOptions(t *testing.T) { externalURL: "https://myhost.example.com:8443", Host: "0.0.0.0", Port: 443, + Scheme: "https", event: EventConfig{ pubsubProjectID: "test-project", pubsubTopic: "gcs-events", @@ -169,6 +378,7 @@ func TestToFakeGcsOptions(t *testing.T) { ExternalURL: "https://myhost.example.com:8443", Host: "0.0.0.0", Port: 443, + Scheme: "https", EventOptions: notification.EventManagerOptions{ ProjectID: "test-project", TopicName: "gcs-events", @@ -193,6 +403,7 @@ func TestToFakeGcsOptions(t *testing.T) { externalURL: "https://myhost.example.com:8443", Host: "0.0.0.0", Port: 443, + Scheme: "https", }, fakestorage.Options{ StorageRoot: "", @@ -200,6 +411,7 @@ func TestToFakeGcsOptions(t *testing.T) { ExternalURL: "https://myhost.example.com:8443", Host: "0.0.0.0", Port: 443, + Scheme: "https", NoListener: true, }, }, @@ -209,7 +421,7 @@ func TestToFakeGcsOptions(t *testing.T) { test := test t.Run(test.name, func(t *testing.T) { t.Parallel() - opts := test.config.ToFakeGcsOptions() + opts := test.config.ToFakeGcsOptions(test.config.Scheme) ignWriter := cmpopts.IgnoreFields(fakestorage.Options{}, "Writer") if diff := cmp.Diff(opts, test.expected, ignWriter); diff != "" { t.Errorf("wrong set of options returned\nwant %#v\ngot %#v\ndiff: %v", test.expected, opts, diff) diff --git a/main.go b/main.go index 83c30bee16..437444290d 100644 --- a/main.go +++ b/main.go @@ -24,27 +24,16 @@ import ( "github.com/sirupsen/logrus" ) -func main() { - cfg, err := config.Load(os.Args[1:]) - if err == flag.ErrHelp { - return - } - if err != nil { - log.Fatal(err) - } +func createListener(logger *logrus.Logger, cfg *config.Config, scheme string) (net.Listener, *fakestorage.Options) { + opts := cfg.ToFakeGcsOptions(scheme) - logger := logrus.New() - logger.SetLevel(cfg.LogLevel) - - opts := cfg.ToFakeGcsOptions() - - addr := fmt.Sprintf("%s:%d", cfg.Host, cfg.Port) + addr := fmt.Sprintf("%s:%d", opts.Host, opts.Port) listener, err := net.Listen("tcp", addr) if err != nil { - log.Fatal(err) + logger.Fatal(err) } - if cfg.Scheme == "https" { + if opts.Scheme == "https" { var tlsConfig *tls.Config if opts.CertificateLocation != "" && opts.PrivateKeyLocation != "" { cert, err := tls.LoadX509KeyPair(opts.CertificateLocation, opts.PrivateKeyLocation) @@ -62,26 +51,68 @@ func main() { listener = tls.NewListener(listener, tlsConfig) } + return listener, &opts +} + +func startServer(logger *logrus.Logger, cfg *config.Config) { + type listenerAndOpts struct { + listener net.Listener + opts *fakestorage.Options + } + + var listenersAndOpts []listenerAndOpts + + if cfg.Scheme != "both" { + listener, opts := createListener(logger, cfg, cfg.Scheme) + listenersAndOpts = []listenerAndOpts{{listener, opts}} + } else { + httpListener, httpOpts := createListener(logger, cfg, "http") + httpsListener, httpsOpts := createListener(logger, cfg, "https") + listenersAndOpts = []listenerAndOpts{ + {httpListener, httpOpts}, + {httpsListener, httpsOpts}, + } + } + addMimeTypes() - httpServer, err := fakestorage.NewServerWithOptions(opts) + httpServer, err := fakestorage.NewServerWithOptions(*listenersAndOpts[0].opts) if err != nil { logger.WithError(err).Fatal("couldn't start the server") } grpcServer := grpc.NewServerWithBackend(httpServer.Backend()) - go func() { - http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.ProtoMajor == 2 && strings.HasPrefix( - r.Header.Get("Content-Type"), "application/grpc") { - grpcServer.ServeHTTP(w, r) - } else { - httpServer.HTTPHandler().ServeHTTP(w, r) - } - })) - }() - logger.Infof("server started at %s://%s:%d", cfg.Scheme, cfg.Host, cfg.Port) + for _, listenerAndOpts := range listenersAndOpts { + go func(listener net.Listener) { + http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.ProtoMajor == 2 && strings.HasPrefix( + r.Header.Get("Content-Type"), "application/grpc") { + grpcServer.ServeHTTP(w, r) + } else { + httpServer.HTTPHandler().ServeHTTP(w, r) + } + })) + }(listenerAndOpts.listener) + + logger.Infof("server started at %s://%s:%d", + listenerAndOpts.opts.Scheme, listenerAndOpts.opts.Host, listenerAndOpts.opts.Port) + } +} + +func main() { + cfg, err := config.Load(os.Args[1:]) + if err == flag.ErrHelp { + return + } + if err != nil { + log.Fatal(err) + } + + logger := logrus.New() + logger.SetLevel(cfg.LogLevel) + + startServer(logger, &cfg) ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt, syscall.SIGTERM)