apiv2: keyspace management (#5313)

ref #5265, ref #5291, ref #5293, ref #5297 Add APIv2 HTTP interface for keyspace management. Signed-off-by: David <[email protected]> Co-authored-by: Ti Chi Robot <[email protected]>
tikv · Feb 2, 2023 · abb62d2 · abb62d2
1 parent 8830739
commit abb62d2
Show file tree

Hide file tree

Showing 4 changed files with 627 additions and 22 deletions.
diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go
@@ -0,0 +1,341 @@
+// Copyright 2023 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/pingcap/errors"
+	"github.com/pingcap/kvproto/pkg/keyspacepb"
+	"github.com/tikv/pd/pkg/errs"
+	"github.com/tikv/pd/server"
+	"github.com/tikv/pd/server/apiv2/middlewares"
+	"github.com/tikv/pd/server/keyspace"
+)
+
+// RegisterKeyspace register keyspace related handlers to router paths.
+func RegisterKeyspace(r *gin.RouterGroup) {
+	router := r.Group("keyspaces")
+	router.Use(middlewares.BootstrapChecker())
+	router.POST("", CreateKeyspace)
+	router.GET("", LoadAllKeyspaces)
+	router.GET("/:name", LoadKeyspace)
+	router.PATCH("/:name/config", UpdateKeyspaceConfig)
+	router.PUT("/:name/state", UpdateKeyspaceState)
+}
+
+// CreateKeyspaceParams represents parameters needed when creating a new keyspace.
+// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it.
+type CreateKeyspaceParams struct {
+	Name   string            `json:"name"`
+	Config map[string]string `json:"config"`
+}
+
+// CreateKeyspace creates keyspace according to given input.
+// @Tags     keyspaces
+// @Summary  Create new keyspace.
+// @Param    body  body  CreateKeyspaceParams  true  "Create keyspace parameters"
+// @Produce  json
+// @Success  200  {object}  KeyspaceMeta
+// @Failure  400  {string}  string  "The input is invalid."
+// @Failure  500  {string}  string  "PD server failed to proceed the request."
+// @Router   /keyspaces [post]
+func CreateKeyspace(c *gin.Context) {
+	svr := c.MustGet("server").(*server.Server)
+	manager := svr.GetKeyspaceManager()
+	createParams := &CreateKeyspaceParams{}
+	err := c.BindJSON(createParams)
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause())
+		return
+	}
+	req := &keyspace.CreateKeyspaceRequest{
+		Name:   createParams.Name,
+		Config: createParams.Config,
+		Now:    time.Now().Unix(),
+	}
+	meta, err := manager.CreateKeyspace(req)
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+		return
+	}
+	c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta})
+}
+
+// LoadKeyspace returns target keyspace.
+// @Tags     keyspaces
+// @Summary  Get keyspace info.
+// @Param    name  path  string  true  "Keyspace Name"
+// @Produce  json
+// @Success  200  {object}  KeyspaceMeta
+// @Failure  500  {string}  string  "PD server failed to proceed the request."
+// @Router   /keyspaces/{name} [get]
+func LoadKeyspace(c *gin.Context) {
+	svr := c.MustGet("server").(*server.Server)
+	manager := svr.GetKeyspaceManager()
+	name := c.Param("name")
+	meta, err := manager.LoadKeyspace(name)
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+		return
+	}
+	c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta})
+}
+
+// parseLoadAllQuery parses LoadAllKeyspaces' query parameters.
+// page_token:
+// The keyspace id of the scan start. If not set, scan from keyspace with id 1.
+// It's string of spaceID of the previous scan result's last element (next_page_token).
+// limit:
+// The maximum number of keyspace metas to return. If not set, no limit is posed.
+// Every scan scans limit + 1 keyspaces (if limit != 0), the extra scanned keyspace
+// is to check if there's more, and used to set next_page_token in response.
+func parseLoadAllQuery(c *gin.Context) (scanStart uint32, scanLimit int, err error) {
+	pageToken, set := c.GetQuery("page_token")
+	if !set || pageToken == "" {
+		// If pageToken is empty or unset, then scan from spaceID of 1.
+		scanStart = 0
+	} else {
+		scanStart64, err := strconv.ParseUint(pageToken, 10, 32)
+		if err != nil {
+			return 0, 0, err
+		}
+		scanStart = uint32(scanStart64)
+	}
+
+	limitStr, set := c.GetQuery("limit")
+	if !set || limitStr == "" || limitStr == "0" {
+		// If limit is unset or empty or 0, then no limit is posed for scan.
+		scanLimit = 0
+	} else {
+		scanLimit64, err := strconv.ParseInt(limitStr, 10, 64)
+		if err != nil {
+			return 0, 0, err
+		}
+		// Scan an extra element for next_page_token.
+		scanLimit = int(scanLimit64) + 1
+	}
+
+	return scanStart, scanLimit, nil
+}
+
+// LoadAllKeyspacesResponse represents response given when loading all keyspaces.
+// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it.
+type LoadAllKeyspacesResponse struct {
+	Keyspaces []*KeyspaceMeta `json:"keyspaces"`
+	// Token that can be used to read immediate next page.
+	// If it's empty, then end has been reached.
+	NextPageToken string `json:"next_page_token"`
+}
+
+// LoadAllKeyspaces loads range of keyspaces.
+// @Tags     keyspaces
+// @Summary  list keyspaces.
+// @Param    page_token  query  string  false  "page token"
+// @Param    limit       query  string  false  "maximum number of results to return"
+// @Produce  json
+// @Success  200  {object}  LoadAllKeyspacesResponse
+// @Failure  400  {string}  string  "The input is invalid."
+// @Failure  500  {string}  string  "PD server failed to proceed the request."
+// @Router   /keyspaces [get]
+func LoadAllKeyspaces(c *gin.Context) {
+	svr := c.MustGet("server").(*server.Server)
+	manager := svr.GetKeyspaceManager()
+	scanStart, scanLimit, err := parseLoadAllQuery(c)
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())
+		return
+	}
+	scanned, err := manager.LoadRangeKeyspace(scanStart, scanLimit)
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+		return
+	}
+	resp := &LoadAllKeyspacesResponse{}
+	// If scanned 0 keyspaces, return result immediately.
+	if len(scanned) == 0 {
+		c.IndentedJSON(http.StatusOK, resp)
+		return
+	}
+	var resultKeyspaces []*KeyspaceMeta
+	if scanLimit == 0 || len(scanned) < scanLimit {
+		// No next page, all scanned are results.
+		resultKeyspaces = make([]*KeyspaceMeta, len(scanned))
+		for i, meta := range scanned {
+			resultKeyspaces[i] = &KeyspaceMeta{meta}
+		}
+	} else {
+		// Scanned limit + 1 keyspaces, there is next page, all but last are results.
+		resultKeyspaces = make([]*KeyspaceMeta, len(scanned)-1)
+		for i := range resultKeyspaces {
+			resultKeyspaces[i] = &KeyspaceMeta{scanned[i]}
+		}
+		// Also set next_page_token here.
+		resp.NextPageToken = strconv.Itoa(int(scanned[len(scanned)-1].Id))
+	}
+	resp.Keyspaces = resultKeyspaces
+	c.IndentedJSON(http.StatusOK, resp)
+}
+
+// UpdateConfigParams represents parameters needed to modify target keyspace's configs.
+// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it.
+// A Map of string to string pointer is used to differentiate between json null and "",
+// which will both be set to "" if value type is string during binding.
+type UpdateConfigParams struct {
+	Config map[string]*string `json:"config"`
+}
+
+// UpdateKeyspaceConfig updates target keyspace's config.
+// This api uses PATCH semantic and supports JSON Merge Patch.
+// format and processing rules.
+// @Tags     keyspaces
+// @Summary  Update keyspace config.
+// @Param    name  path  string              true  "Keyspace Name"
+// @Param    body  body  UpdateConfigParams  true  "Update keyspace parameters"
+// @Produce  json
+// @Success  200  {object}  KeyspaceMeta
+// @Failure  400  {string}  string  "The input is invalid."
+// @Failure  500  {string}  string  "PD server failed to proceed the request."
+// Router /keyspaces/{name}/config [patch]
+func UpdateKeyspaceConfig(c *gin.Context) {
+	svr := c.MustGet("server").(*server.Server)
+	manager := svr.GetKeyspaceManager()
+	name := c.Param("name")
+	configParams := &UpdateConfigParams{}
+	err := c.BindJSON(configParams)
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause())
+		return
+	}
+	mutations := getMutations(configParams.Config)
+	meta, err := manager.UpdateKeyspaceConfig(name, mutations)
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+		return
+	}
+	c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta})
+}
+
+// getMutations converts a given JSON merge patch to a series of keyspace config mutations.
+func getMutations(patch map[string]*string) []*keyspace.Mutation {
+	mutations := make([]*keyspace.Mutation, 0, len(patch))
+	for k, v := range patch {
+		if v == nil {
+			mutations = append(mutations, &keyspace.Mutation{
+				Op:  keyspace.OpDel,
+				Key: k,
+			})
+		} else {
+			mutations = append(mutations, &keyspace.Mutation{
+				Op:    keyspace.OpPut,
+				Key:   k,
+				Value: *v,
+			})
+		}
+	}
+	return mutations
+}
+
+// UpdateStateParam represents parameters needed to modify target keyspace's state.
+// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it.
+type UpdateStateParam struct {
+	State string `json:"state"`
+}
+
+// UpdateKeyspaceState update the target keyspace's state.
+// @Tags     keyspaces
+// @Summary  Update keyspace state.
+// @Param    name  path  string            true  "Keyspace Name"
+// @Param    body  body  UpdateStateParam  true  "New state for the keyspace"
+// @Produce  json
+// @Success  200  {object}  KeyspaceMeta
+// @Failure  400  {string}  string  "The input is invalid."
+// @Failure  500  {string}  string  "PD server failed to proceed the request."
+// Router /keyspaces/{name}/state [put]
+func UpdateKeyspaceState(c *gin.Context) {
+	svr := c.MustGet("server").(*server.Server)
+	manager := svr.GetKeyspaceManager()
+	name := c.Param("name")
+	param := &UpdateStateParam{}
+	err := c.BindJSON(param)
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause())
+		return
+	}
+	targetState, ok := keyspacepb.KeyspaceState_value[strings.ToUpper(param.State)]
+	if !ok {
+		c.AbortWithStatusJSON(http.StatusBadRequest, errors.Errorf("unknown target state: %s", param.State))
+		return
+	}
+	meta, err := manager.UpdateKeyspaceState(name, keyspacepb.KeyspaceState(targetState), time.Now().Unix())
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+		return
+	}
+	c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta})
+}
+
+// KeyspaceMeta wraps keyspacepb.KeyspaceMeta to provide custom JSON marshal.
+type KeyspaceMeta struct {
+	*keyspacepb.KeyspaceMeta
+}
+
+// MarshalJSON creates custom marshal of KeyspaceMeta with the following:
+// 1. Keyspace ID are removed from marshal result to avoid exposure of internal mechanics.
+// 2. Keyspace State are marshaled to their corresponding name for better readability.
+func (meta *KeyspaceMeta) MarshalJSON() ([]byte, error) {
+	return json.Marshal(&struct {
+		Name           string            `json:"name,omitempty"`
+		State          string            `json:"state,omitempty"`
+		CreatedAt      int64             `json:"created_at,omitempty"`
+		StateChangedAt int64             `json:"state_changed_at,omitempty"`
+		Config         map[string]string `json:"config,omitempty"`
+	}{
+		meta.Name,
+		meta.State.String(),
+		meta.CreatedAt,
+		meta.StateChangedAt,
+		meta.Config,
+	})
+}
+
+// UnmarshalJSON reverse KeyspaceMeta's the Custom JSON marshal.
+func (meta *KeyspaceMeta) UnmarshalJSON(data []byte) error {
+	aux := &struct {
+		Name           string            `json:"name,omitempty"`
+		State          string            `json:"state,omitempty"`
+		CreatedAt      int64             `json:"created_at,omitempty"`
+		StateChangedAt int64             `json:"state_changed_at,omitempty"`
+		Config         map[string]string `json:"config,omitempty"`
+	}{}
+
+	if err := json.Unmarshal(data, aux); err != nil {
+		return err
+	}
+	pbMeta := &keyspacepb.KeyspaceMeta{
+		Name:           aux.Name,
+		State:          keyspacepb.KeyspaceState(keyspacepb.KeyspaceState_value[aux.State]),
+		CreatedAt:      aux.CreatedAt,
+		StateChangedAt: aux.StateChangedAt,
+		Config:         aux.Config,
+	}
+	meta.KeyspaceMeta = pbMeta
+	return nil
+}
diff --git a/server/apiv2/router.go b/server/apiv2/router.go
@@ -22,6 +22,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/joho/godotenv"
 	"github.com/tikv/pd/server"
+	"github.com/tikv/pd/server/apiv2/handlers"
 	"github.com/tikv/pd/server/apiv2/middlewares"
 )
 
@@ -37,6 +38,15 @@ var group = server.APIServiceGroup{
 const apiV2Prefix = "/pd/api/v2/"
 
 // NewV2Handler creates a HTTP handler for API.
+// @title          Placement Driver Core API
+// @version        2.0
+// @description    This is placement driver.
+// @contact.name   Placement Driver Support
+// @contact.url    https://github.com/tikv/pd/issues
+// @contact.email  [email protected]
+// @license.name   Apache 2.0
+// @license.url    http://www.apache.org/licenses/LICENSE-2.0.html
+// @BasePath       /pd/api/v2
 func NewV2Handler(_ context.Context, svr *server.Server) (http.Handler, server.APIServiceGroup, error) {
 	once.Do(func() {
 		// See https://github.com/pingcap/tidb-dashboard/blob/f8ecb64e3d63f4ed91c3dca7a04362418ade01d8/pkg/apiserver/apiserver.go#L84
@@ -50,7 +60,7 @@ func NewV2Handler(_ context.Context, svr *server.Server) (http.Handler, server.A
 		c.Next()
 	})
 	router.Use(middlewares.Redirector())
-	_ = router.Group(apiV2Prefix)
-
+	root := router.Group(apiV2Prefix)
+	handlers.RegisterKeyspace(root)
 	return router, group, nil
 }