From a761b29e73828d3cfcade57c202ce93dda81a0ee Mon Sep 17 00:00:00 2001 From: p4u Date: Mon, 11 Mar 2024 13:34:54 +0100 Subject: [PATCH] api: add a new endpoint to publish a census async Since the publish of the census can take some time, allow the API clients to execute the operation async. So two new endpoints are introduced: /censuses/:censusId/publish/async /censuses/:censusId/check Signed-off-by: p4u --- api/api.go | 3 + api/censuses.go | 139 ++++++++++++++++++++++++----- api/errors.go | 1 + apiclient/census.go | 23 ++++- dockerfiles/testsuite/env.gateway0 | 2 +- 5 files changed, 142 insertions(+), 26 deletions(-) diff --git a/api/api.go b/api/api.go index f021e5765..68cf0879e 100644 --- a/api/api.go +++ b/api/api.go @@ -10,6 +10,7 @@ import ( "fmt" "path/filepath" "strings" + "sync" "go.vocdoni.io/dvote/api/censusdb" "go.vocdoni.io/dvote/data" @@ -69,6 +70,8 @@ type API struct { vocinfo *vochaininfo.VochainInfo censusdb *censusdb.CensusDB db db.Database // used for internal db operations + + censusPublishStatusMap sync.Map // used to store the status of the census publishing process when async } // NewAPI creates a new instance of the API. Attach must be called next. diff --git a/api/censuses.go b/api/censuses.go index dc2f8fea4..05e3e8669 100644 --- a/api/censuses.go +++ b/api/censuses.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "strings" "time" @@ -111,6 +112,22 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } + if err := a.Endpoint.RegisterMethod( + "/censuses/{censusID}/publish/async", + "POST", + apirest.MethodAccessTypePublic, + a.censusPublishHandler, + ); err != nil { + return err + } + if err := a.Endpoint.RegisterMethod( + "/censuses/{censusID}/check", + "GET", + apirest.MethodAccessTypePublic, + a.censusPublishCheckHandler, + ); err != nil { + return err + } if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}/publish/{root}", "POST", @@ -615,7 +632,7 @@ func (a *API) censusDeleteHandler(msg *apirest.APIdata, ctx *httprouter.HTTPCont // @Success 200 {object} object{census=object{censusID=string,uri=string}} "It return published censusID and the ipfs uri where its uploaded" // @Param censusID path string true "Census id" // @Router /censuses/{censusID}/publish [post] -// /censuses/{censusID}/publish/{root} [post] Endpoint docs generated on docs/models/model.go +// @Router /censuses/{censusID}/publish/async [post] func (a *API) censusPublishHandler(msg *apirest.APIdata, ctx *httprouter.HTTPContext) error { token, err := uuid.Parse(msg.AuthToken) if err != nil { @@ -626,6 +643,10 @@ func (a *API) censusPublishHandler(msg *apirest.APIdata, ctx *httprouter.HTTPCon return err } + // check if the request is async + url := strings.Split(ctx.Request.URL.Path, "/") + async := url[len(url)-1] == "async" + ref, err := a.censusdb.Load(censusID, &token) defer a.censusdb.UnLoad() if err != nil { @@ -665,6 +686,11 @@ func (a *API) censusPublishHandler(msg *apirest.APIdata, ctx *httprouter.HTTPCon } return err } + // if async, store the URI in the map for the check endpoint + if async { + a.censusPublishStatusMap.Store(hex.EncodeToString(root), ref.URI) + } + var data []byte if data, err = json.Marshal(&Census{ CensusID: root, @@ -675,36 +701,64 @@ func (a *API) censusPublishHandler(msg *apirest.APIdata, ctx *httprouter.HTTPCon return ctx.Send(data, apirest.HTTPstatusOK) } - // dump the current tree to import them after - dump, err := ref.Tree().Dump() - if err != nil { - return err - } - - // export the tree to the remote storage (IPFS) - uri := "" - if a.storage != nil { - exportData, err := censusdb.BuildExportDump(root, dump, - models.Census_Type(ref.CensusType), ref.MaxLevels) + publishCensus := func() (string, error) { + // dump the current tree to import them after + dump, err := ref.Tree().Dump() if err != nil { - return err + return "", err } - sctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - cid, err := a.storage.Publish(sctx, exportData) + + // export the tree to the remote storage (IPFS) + uri := "" + if a.storage != nil { + exportData, err := censusdb.BuildExportDump(root, dump, + models.Census_Type(ref.CensusType), ref.MaxLevels) + if err != nil { + return "", err + } + sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cid, err := a.storage.Publish(sctx, exportData) + if err != nil { + log.Errorf("could not export tree to storage: %v", err) + } else { + uri = a.storage.URIprefix() + cid + } + } + + newRef, err := a.censusdb.New(root, models.Census_Type(ref.CensusType), uri, + nil, ref.MaxLevels) if err != nil { - log.Errorf("could not export tree to storage: %v", err) - } else { - uri = a.storage.URIprefix() + cid + return "", err } + return uri, newRef.Tree().ImportDump(dump) } - newRef, err := a.censusdb.New(root, models.Census_Type(ref.CensusType), uri, - nil, ref.MaxLevels) - if err != nil { - return err + if async { + a.censusPublishStatusMap.Store(hex.EncodeToString(root), "") + + go func() { + uri, err := publishCensus() + if err != nil { + log.Errorw(err, "could not publish census") + a.censusPublishStatusMap.Store(hex.EncodeToString(root), fmt.Sprintf("error: %v", err.Error())) + return + } + a.censusPublishStatusMap.Store(hex.EncodeToString(root), uri) + }() + + var data []byte + if data, err = json.Marshal(&Census{ + CensusID: root, + }); err != nil { + return err + } + + return ctx.Send(data, apirest.HTTPstatusOK) } - if err := newRef.Tree().ImportDump(dump); err != nil { + + uri, err := publishCensus() + if err != nil { return err } @@ -719,6 +773,43 @@ func (a *API) censusPublishHandler(msg *apirest.APIdata, ctx *httprouter.HTTPCon return ctx.Send(data, apirest.HTTPstatusOK) } +// censusPublishCheckHandler +// +// @Summary Check census publish status +// @Description.markdown censusPublishCheckHandler +// @Tags Censuses +// @Produce json +// @Success 200 {object} object{census=object{censusID=string,uri=string}} "It return published censusID and the ipfs uri where its uploaded" +// @Param censusID path string true "Census id" +// @Router /censuses/{censusID}/check [get] +func (a *API) censusPublishCheckHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + censusID, err := censusIDparse(ctx.URLParam("censusID")) + if err != nil { + return err + } + uriOrErrorAny, ok := a.censusPublishStatusMap.Load(hex.EncodeToString(censusID)) + if !ok { + return ErrCensusNotFound + } + log.Warnf("census publish status: %v", uriOrErrorAny) + uriOrError := uriOrErrorAny.(string) + if uriOrError == "" { + return ctx.Send(nil, apirest.HTTPstatusNoContent) + } + if strings.HasPrefix(uriOrError, "error:") { + return ErrCensusBuild.With(uriOrError[7:]) + } + var data []byte + if data, err = json.Marshal(&Census{ + CensusID: censusID, + URI: uriOrError, + }); err != nil { + return err + } + a.censusPublishStatusMap.Delete(hex.EncodeToString(censusID)) + return ctx.Send(data, apirest.HTTPstatusOK) +} + // censusProofHandler // // @Summary Prove key to census diff --git a/api/errors.go b/api/errors.go index 1a86b760f..6346b0017 100644 --- a/api/errors.go +++ b/api/errors.go @@ -112,4 +112,5 @@ var ( ErrCantCountVotes = apirest.APIerror{Code: 5029, HTTPstatus: apirest.HTTPstatusInternalErr, Err: fmt.Errorf("cannot count votes")} ErrVochainOverloaded = apirest.APIerror{Code: 5030, HTTPstatus: apirest.HTTPstatusServiceUnavailable, Err: fmt.Errorf("vochain overloaded")} ErrGettingSIK = apirest.APIerror{Code: 5031, HTTPstatus: apirest.HTTPstatusInternalErr, Err: fmt.Errorf("error getting SIK")} + ErrCensusBuild = apirest.APIerror{Code: 5032, HTTPstatus: apirest.HTTPstatusInternalErr, Err: fmt.Errorf("error building census")} ) diff --git a/apiclient/census.go b/apiclient/census.go index 7ade7215a..bd4aadff6 100644 --- a/apiclient/census.go +++ b/apiclient/census.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "math/big" + "time" "go.vocdoni.io/dvote/api" "go.vocdoni.io/dvote/httprouter/apirest" @@ -79,17 +80,37 @@ func (c *HTTPclient) CensusSize(censusID types.HexBytes) (uint64, error) { // CensusPublish publishes a census to the distributed data storage and returns its root hash // and storage URI. func (c *HTTPclient) CensusPublish(censusID types.HexBytes) (types.HexBytes, string, error) { - resp, code, err := c.Request(HTTPPOST, nil, "censuses", censusID.String(), "publish") + resp, code, err := c.Request(HTTPPOST, nil, "censuses", censusID.String(), "publish", "async") if err != nil { return nil, "", err } if code != apirest.HTTPstatusOK { return nil, "", fmt.Errorf("%s: %d (%s)", errCodeNot200, code, resp) } + censusData := &api.Census{} if err := json.Unmarshal(resp, censusData); err != nil { return nil, "", fmt.Errorf("could not unmarshal response: %w", err) } + + // wait for the census to be ready and get the root hash and storage URI + for { + time.Sleep(2 * time.Second) + resp, code, err := c.Request(HTTPGET, nil, "censuses", censusData.CensusID.String(), "check") + if err != nil { + return nil, "", err + } + if code == apirest.HTTPstatusOK { + if err := json.Unmarshal(resp, censusData); err != nil { + return nil, "", fmt.Errorf("could not unmarshal response: %w", err) + } + break + } + if code == apirest.HTTPstatusNoContent { + continue + } + return nil, "", fmt.Errorf("%s: %d (%s)", errCodeNot200, code, resp) + } return censusData.CensusID, censusData.URI, nil } diff --git a/dockerfiles/testsuite/env.gateway0 b/dockerfiles/testsuite/env.gateway0 index 5f5a1344f..a1bec70af 100755 --- a/dockerfiles/testsuite/env.gateway0 +++ b/dockerfiles/testsuite/env.gateway0 @@ -1,7 +1,7 @@ VOCDONI_DATADIR=/app/run VOCDONI_MODE=gateway VOCDONI_LOGLEVEL=debug -VOCDONI_VOCHAIN_LOGLEVEL=info +VOCDONI_VOCHAIN_LOGLEVEL=error VOCDONI_DEV=True VOCDONI_ENABLEAPI=True VOCDONI_ENABLERPC=True