Skip to content

Commit

Permalink
Implements image storage
Browse files Browse the repository at this point in the history
- Add image upload/download endpoints with authentication
- Create object storage client for file handling
- Integrate storage functionality with user system
- Update API config and database schema
- Add environment variables for storage configuration
  • Loading branch information
emmdim committed Dec 20, 2024
1 parent 888bb37 commit 14f1ffd
Show file tree
Hide file tree
Showing 13 changed files with 365 additions and 4 deletions.
12 changes: 12 additions & 0 deletions api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/vocdoni/saas-backend/account"
"github.com/vocdoni/saas-backend/db"
"github.com/vocdoni/saas-backend/notifications"
"github.com/vocdoni/saas-backend/objectstorage"
"github.com/vocdoni/saas-backend/stripe"
"github.com/vocdoni/saas-backend/subscriptions"
"go.vocdoni.io/dvote/apiclient"
Expand Down Expand Up @@ -40,6 +41,8 @@ type APIConfig struct {
StripeClient *stripe.StripeClient
// Subscriptions permissions manager
Subscriptions *subscriptions.Subscriptions
// Object storage
ObjectStorage *objectstorage.ObjectStorageClient
}

// API type represents the API HTTP server with JWT authentication capabilities.
Expand All @@ -57,6 +60,7 @@ type API struct {
transparentMode bool
stripe *stripe.StripeClient
subscriptions *subscriptions.Subscriptions
objectStorage *objectstorage.ObjectStorageClient
}

// New creates a new API HTTP server. It does not start the server. Use Start() for that.
Expand All @@ -77,6 +81,7 @@ func New(conf *APIConfig) *API {
transparentMode: conf.FullTransparentMode,
stripe: conf.StripeClient,
subscriptions: conf.Subscriptions,
objectStorage: conf.ObjectStorage,
}
}

Expand Down Expand Up @@ -203,6 +208,13 @@ func (a *API) initRouter() http.Handler {
r.Get(planInfoEndpoint, a.planInfoHandler)
log.Infow("new route", "method", "POST", "path", subscriptionsWebhook)
r.Post(subscriptionsWebhook, a.handleWebhook)
// TODO move to private after testing
// upload an image to the object storage
log.Infow("new route", "method", "POST", "path", objectStorageUploadTypedEndpoint)
r.Post(objectStorageUploadTypedEndpoint, a.uploadImageWithFormHandler)
// upload an image to the object storage
log.Infow("new route", "method", "GET", "path", objectStorageDownloadTypedEndpoint)
r.Get(objectStorageDownloadTypedEndpoint, a.downloadImageInlineHandler)
})
a.router = r
return r
Expand Down
36 changes: 36 additions & 0 deletions api/docs.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
- [🏦 Plans](#-plans)
- [🛒 Get Available Plans](#-get-plans)
- [🛍️ Get Plan Info](#-get-plan-info)
- [ Storage](#-storage)
- [ Upload image from origin](#-upload-image-with-origin)

</details>

Expand Down Expand Up @@ -916,3 +918,37 @@ This request can be made only by organization admins.
| `400` | `40010` | `malformed URL parameter` |
| `400` | `40023` | `plan not found` |
| `500` | `50002` | `internal server error` |


## Storage

### Upload image with origin

* **Path** `/storage/{origin}`
* **Method** `POST`

Accepting files uploaded by forms as such:
```html
<form action="http://localhost:8000" method="post" enctype="multipart/form-data">
<p><input type="text" name="text" value="text default">
<p><input type="file" name="file1">
<p><input type="file" name="file2">
<p><button type="submit">Submit</button>
</form>
```

* **Response**

This methods uploads the images/files to 3rd party object storages and returns the URI where they are publicy available in inline mode.
```json
{
"urls": ["https://file1.store.com","https://file1.store.com"]
}
```

* **Errors**

| HTTP Status | Error code | Message |
|:---:|:---:|:---|
| `401` | `40001` | `user not authorized` |
| `500` | `50002` | `internal server error` |
73 changes: 73 additions & 0 deletions api/object_storage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package api

import (
"net/http"

"github.com/go-chi/chi/v5"
)

func (a *API) uploadImageWithFormHandler(w http.ResponseWriter, r *http.Request) {
// check if the user is authenticated
// get the user from the request context
user, ok := userFromContext(r.Context())
if !ok {
ErrUnauthorized.Write(w)
return
}

// 32 MB is the default used by FormFile() function
if err := r.ParseMultipartForm(32 << 20); err != nil {
ErrGenericInternalServerError.With("could not parse form").Write(w)
return
}

// Get a reference to the fileHeaders.
// They are accessible only after ParseMultipartForm is called
files := r.MultipartForm.File["file"]
var returnURLs []string
for _, fileHeader := range files {
// Open the file
file, err := fileHeader.Open()
if err != nil {
ErrGenericInternalServerError.Withf("cannot open file %s", err.Error()).Write(w)
break
}
defer func() {
if err := file.Close(); err != nil {
ErrGenericInternalServerError.Withf("cannot close file %s", err.Error()).Write(w)
return
}
}()
// upload the file using the object storage client
// and get the URL of the uploaded file
url, err := a.objectStorage.Put(file, fileHeader.Size, user.Email)
if err != nil {
ErrGenericInternalServerError.Withf("cannot upload file %s", err.Error()).Write(w)
break
}
returnURLs = append(returnURLs, url)
}
httpWriteJSON(w, map[string][]string{"urls": returnURLs})
}

func (a *API) downloadImageInlineHandler(w http.ResponseWriter, r *http.Request) {
objectID := chi.URLParam(r, "objectName")
if objectID == "" {
ErrMalformedURLParam.With("objectID is required").Write(w)
return
}
// get the object from the object storage client
object, err := a.objectStorage.Get(objectID)
if err != nil {
ErrGenericInternalServerError.Withf("cannot get object %s", err.Error()).Write(w)
return
}
// write the object to the response
w.Header().Set("Content-Type", object.ContentType)
// w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data)))
w.Header().Set("Content-Disposition", "inline")
if _, err := w.Write(object.Data); err != nil {
ErrGenericInternalServerError.Withf("cannot write object %s", err.Error()).Write(w)
return
}
}
4 changes: 3 additions & 1 deletion api/plans.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package api
import (
"net/http"
"strconv"

"github.com/go-chi/chi/v5"
)

// getSubscriptionsHandler handles the request to get the subscriptions of an organization.
Expand All @@ -20,7 +22,7 @@ func (a *API) getPlansHandler(w http.ResponseWriter, r *http.Request) {

func (a *API) planInfoHandler(w http.ResponseWriter, r *http.Request) {
// get the plan ID from the URL
planID := r.URL.Query().Get("planID")
planID := chi.URLParam(r, "planID")
// check the the planID is not empty
if planID == "" {
ErrMalformedURLParam.Withf("planID is required").Write(w)
Expand Down
6 changes: 6 additions & 0 deletions api/routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,4 +66,10 @@ const (
planInfoEndpoint = "/plans/{planID}"
// POST /subscriptions/webhook to receive the subscription webhook from stripe
subscriptionsWebhook = "/subscriptions/webhook"

// object storage routes
// POST /storage/{origin} to upload an image to the object storage
objectStorageUploadTypedEndpoint = "/storage"
// GET /storage/{origin}/{filename} to download an image from the object storage
objectStorageDownloadTypedEndpoint = "/storage/{objectName}"
)
3 changes: 2 additions & 1 deletion api/users.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"net/http"
"time"

"github.com/go-chi/chi/v5"
"github.com/vocdoni/saas-backend/db"
"github.com/vocdoni/saas-backend/internal"
"github.com/vocdoni/saas-backend/notifications/mailtemplates"
Expand Down Expand Up @@ -174,7 +175,7 @@ func (a *API) verifyUserAccountHandler(w http.ResponseWriter, r *http.Request) {
// returned.
func (a *API) userVerificationCodeInfoHandler(w http.ResponseWriter, r *http.Request) {
// get the user email of the user from the request query
userEmail := r.URL.Query().Get("email")
userEmail := chi.URLParam(r, "email")
// check the email is not empty
if userEmail == "" {
ErrInvalidUserData.With("no email provided").Write(w)
Expand Down
10 changes: 10 additions & 0 deletions cmd/service/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,18 @@ import (
"github.com/vocdoni/saas-backend/db"
"github.com/vocdoni/saas-backend/notifications/mailtemplates"
"github.com/vocdoni/saas-backend/notifications/smtp"
"github.com/vocdoni/saas-backend/objectstorage"
"github.com/vocdoni/saas-backend/stripe"
"github.com/vocdoni/saas-backend/subscriptions"
"go.vocdoni.io/dvote/apiclient"
"go.vocdoni.io/dvote/log"
)

var serverURL = "http://localhost:8080"

func main() {
// define flags
flag.String("server", serverURL, "The full URL of the server (http or https)")
flag.StringP("host", "h", "0.0.0.0", "listen address")
flag.IntP("port", "p", 8080, "listen port")
flag.StringP("secret", "s", "", "API secret")
Expand All @@ -46,6 +50,7 @@ func main() {
}
viper.AutomaticEnv()
// read the configuration
server := viper.GetString("server")
host := viper.GetString("host")
port := viper.GetInt("port")
apiEndpoint := viper.GetString("vocdoniApi")
Expand Down Expand Up @@ -144,6 +149,11 @@ func main() {
DB: database,
})
apiConf.Subscriptions = subscriptions
// initialize the s3 like object storage
apiConf.ObjectStorage = objectstorage.New(&objectstorage.ObjectStorageConfig{
DB: database,
ServerURL: server,
})
// create the local API server
api.New(apiConf).Start()
log.Infow("server started", "host", host, "port", port)
Expand Down
12 changes: 12 additions & 0 deletions db/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ func (ms *MongoStorage) initCollections(database string) error {
if ms.plans, err = getCollection("plans"); err != nil {
return err
}
// objects collection
if ms.objects, err = getCollection("objects"); err != nil {
return err
}
return nil
}

Expand Down Expand Up @@ -176,6 +180,14 @@ func (ms *MongoStorage) createIndexes() error {
}); err != nil {
return fmt.Errorf("failed to create index on invitationCode for organization invites: %w", err)
}
// add objectID index to objects collection
// objectIDIndex := mongo.IndexModel{
// Keys: bson.D{{Key: "_id", Value: 1}}, // 1 for ascending order
// Options: options.Index().SetUnique(true),
// }
// if _, err := ms.objects.Indexes().CreateOne(ctx, objectIDIndex); err != nil {
// return fmt.Errorf("failed to create index on _id for objects: %w", err)
// }
return nil
}

Expand Down
5 changes: 5 additions & 0 deletions db/mongo.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type MongoStorage struct {
organizations *mongo.Collection
organizationInvites *mongo.Collection
plans *mongo.Collection
objects *mongo.Collection
}

type Options struct {
Expand Down Expand Up @@ -119,6 +120,10 @@ func (ms *MongoStorage) Reset() error {
if err := ms.plans.Drop(ctx); err != nil {
return err
}
// drop the objects collection
if err := ms.objects.Drop(ctx); err != nil {
return err
}
// init the collections
if err := ms.initCollections(ms.database); err != nil {
return err
Expand Down
68 changes: 68 additions & 0 deletions db/object.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package db

import (
"context"
"fmt"
"time"

"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)

func (ms *MongoStorage) Object(id string) (*Object, error) {
ms.keysLock.RLock()
defer ms.keysLock.RUnlock()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()

// find the object in the database
result := ms.objects.FindOne(ctx, bson.M{"_id": id})
obj := &Object{}
if err := result.Decode(obj); err != nil {
if err == mongo.ErrNoDocuments {
return nil, ErrNotFound
}
return nil, err
}
return obj, nil
}

// SetObject sets the object data for the given objectID. If the
// object does not exist, it will be created with the given data, otherwise it
// will be updated.
func (ms *MongoStorage) SetObject(objectID, userID, contentType string, data []byte) error {
object := &Object{
ID: objectID,
Data: data,
CreatedAt: time.Now(),
UserID: userID,
ContentType: contentType,
}
ms.keysLock.Lock()
defer ms.keysLock.Unlock()
return ms.setObject(object)
}

// RemoveObject removes the object data for the given objectID.
func (ms *MongoStorage) RemoveObject(objectID string) error {
ms.keysLock.Lock()
defer ms.keysLock.Unlock()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := ms.objects.DeleteOne(ctx, bson.M{"_id": objectID})
return err
}

func (ms *MongoStorage) setObject(object *Object) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
opts := options.ReplaceOptions{}
opts.Upsert = new(bool)
*opts.Upsert = true
_, err := ms.objects.ReplaceOne(ctx, bson.M{"_id": object.ID}, object, &opts)
if err != nil {
return fmt.Errorf("cannot update object: %w", err)
}
return err
}
11 changes: 11 additions & 0 deletions db/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,3 +134,14 @@ type OrganizationInvite struct {
Role UserRole `json:"role" bson:"role"`
Expiration time.Time `json:"expiration" bson:"expiration"`
}

// Object represents a user uploaded object Includes user defined ID and the data
// as a byte array.
type Object struct {
ID string `json:"id" bson:"_id"`
Name string `json:"name" bson:"name"`
Data []byte `json:"data" bson:"data"`
CreatedAt time.Time `json:"createdAt" bson:"createdAt"`
UserID string `json:"userId" bson:"userId"`
ContentType string `json:"contentType" bson:"contentType"`
}
6 changes: 4 additions & 2 deletions example.env
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
VOCDONI_SERVERURL=http://localhost:8080
VOCDONI_PORT=8080
VOCDONI_SECRET=supersecret
VOCDONI_PRIVATEKEY=vochain-private-key
Expand All @@ -6,5 +7,6 @@ VOCDONI_SMTPUSERNAME=admin
VOCDONI_SMTPPASSWORD=password
VOCDONI_EMAILFROMADDRESS=[email protected]
VOCDONI_EMAILFROMADDRESS=[email protected]
STRIPE_API_SECRET=stripe_key
STRIPE_WEBHOOK_SECRET=stripe_webhook_key
VOCDONI_STRIPEAPISECRET=test
VOCDONI_STRIPEWEBHOOKSEC=test
VOCDONI_WEBURL=test
Loading

0 comments on commit 14f1ffd

Please sign in to comment.