From 54a9a53baff538ebe829bf1c645a7997ac6f0808 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 5 Jul 2022 13:08:58 +0800 Subject: [PATCH 01/76] storage: keyspace storage Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- go.mod | 3 + go.sum | 13 +--- server/storage/endpoint/key_path.go | 17 +++++ server/storage/endpoint/keyspace.go | 67 ++++++++++++++++++ server/storage/keyspace_test.go | 106 ++++++++++++++++++++++++++++ server/storage/storage.go | 1 + 6 files changed, 196 insertions(+), 11 deletions(-) create mode 100644 server/storage/endpoint/keyspace.go create mode 100644 server/storage/keyspace_test.go diff --git a/go.mod b/go.mod index 6c2a3c4380b..0e9929ea982 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,9 @@ module github.com/tikv/pd go 1.16 +// TODO: Remove after kvproto merge +replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220704092113-f8d176ab7cb4 + require ( github.com/AlekSi/gocov-xml v1.0.0 github.com/BurntSushi/toml v0.3.1 diff --git a/go.sum b/go.sum index 54862125e1a..9e0721ce726 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AlekSi/gocov-xml v1.0.0 h1:4QctJBgXEkbzeKz6PJy6bt3JSPNSN4I2mITYW+eKUoQ= github.com/AlekSi/gocov-xml v1.0.0/go.mod h1:J0qYeZ6tDg4oZubW9mAAgxlqw39PDfoEkzB3HXSbEuA= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220704092113-f8d176ab7cb4 h1:SNXqBJDGf3s7OrgyhUtry9vZoD4IpTI9fNKvGWqcN/s= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220704092113-f8d176ab7cb4/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -173,10 +175,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/goccy/go-graphviz v0.0.9 h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IPQQ= github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= @@ -190,7 +190,6 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -298,7 +297,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -407,10 +405,6 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMtVcOkjUcuQKh+YrluSo7+7YMCQSzy30= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= -github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= @@ -719,7 +713,6 @@ golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 h1:M73Iuj3xbbb9Uk1DYhzydths golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -760,11 +753,9 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index db7032b654b..8f29a165138 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -17,6 +17,7 @@ package endpoint import ( "fmt" "path" + "strconv" ) const ( @@ -34,6 +35,9 @@ const ( minResolvedTS = "min_resolved_ts" keySpaceSafePointPrefix = "key_space/gc_safepoint" keySpaceGCSafePointSuffix = "gc" + keyspacePrefix = "keyspaces" + keyspaceMeta = "meta" + spaceIDBase = 16 ) // AppendToRootPath appends the given key to the rootPath. @@ -136,3 +140,16 @@ func KeySpaceSafePointPrefix() string { func KeySpaceGCSafePointSuffix() string { return "/" + keySpaceGCSafePointSuffix } + +// KeyspaceMetaPrefix returns the prefix of keyspaces' metadata. +// Prefix: keyspaces/meta/ +func KeyspaceMetaPrefix() string { + return path.Join(keyspacePrefix, keyspaceMeta) + "/" +} + +// KeyspaceMetaPath returns the path to the given keyspace's metadata. +// Path: /keyspaces/meta/{space_id} +func KeyspaceMetaPath(spaceID uint32) string { + idStr := strconv.FormatUint(uint64(spaceID), spaceIDBase) + return path.Join(KeyspaceMetaPrefix(), idStr) +} diff --git a/server/storage/endpoint/keyspace.go b/server/storage/endpoint/keyspace.go new file mode 100644 index 00000000000..fd1b4cf5fc9 --- /dev/null +++ b/server/storage/endpoint/keyspace.go @@ -0,0 +1,67 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package endpoint + +import ( + "github.com/gogo/protobuf/proto" + "github.com/pingcap/kvproto/pkg/keyspacepb" + "go.etcd.io/etcd/clientv3" +) + +type KeyspaceStorage interface { + // SaveKeyspace saves the given keyspace to the storage. + SaveKeyspace(*keyspacepb.KeyspaceMeta) error + // LoadKeyspace loads keyspace specified by spaceID. + LoadKeyspace(spaceID uint32, keyspace *keyspacepb.KeyspaceMeta) (bool, error) + // LoadRangeKeyspace loads no more than limit keyspaces starting at startID. + LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) +} + +var _ KeyspaceStorage = (*StorageEndpoint)(nil) + +// SaveKeyspace saves the given keyspace to the storage. +func (se *StorageEndpoint) SaveKeyspace(keyspace *keyspacepb.KeyspaceMeta) error { + key := KeyspaceMetaPath(keyspace.GetId()) + return se.saveProto(key, keyspace) +} + +// LoadKeyspace loads keyspace specified by spaceID. +func (se *StorageEndpoint) LoadKeyspace(spaceID uint32, keyspace *keyspacepb.KeyspaceMeta) (bool, error) { + key := KeyspaceMetaPath(spaceID) + return se.loadProto(key, keyspace) +} + +// LoadRangeKeyspace loads keyspaces starting at startID. +// limit specifies the limit of loaded keyspaces. +func (se *StorageEndpoint) LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { + startKey := KeyspaceMetaPath(startID) + endKey := clientv3.GetPrefixRangeEnd(KeyspaceMetaPrefix()) + keys, values, err := se.LoadRange(startKey, endKey, limit) + if err != nil { + return nil, err + } + if len(keys) == 0 { + return []*keyspacepb.KeyspaceMeta{}, nil + } + keyspaces := make([]*keyspacepb.KeyspaceMeta, 0, len(keys)) + for _, value := range values { + keyspace := &keyspacepb.KeyspaceMeta{} + if err = proto.Unmarshal([]byte(value), keyspace); err != nil { + return nil, err + } + keyspaces = append(keyspaces, keyspace) + } + return keyspaces, nil +} diff --git a/server/storage/keyspace_test.go b/server/storage/keyspace_test.go new file mode 100644 index 00000000000..a51695deb5e --- /dev/null +++ b/server/storage/keyspace_test.go @@ -0,0 +1,106 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "testing" + "time" + + "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/stretchr/testify/require" +) + +func TestSaveLoadKeyspace(t *testing.T) { + re := require.New(t) + storage := NewStorageWithMemoryBackend() + + keyspaces := testKeyspaces() + for _, keyspace := range keyspaces { + re.NoError(storage.SaveKeyspace(keyspace)) + } + + for _, keyspace := range keyspaces { + spaceID := keyspace.GetId() + loadedKeyspace := &keyspacepb.KeyspaceMeta{} + success, err := storage.LoadKeyspace(spaceID, loadedKeyspace) + re.True(success) + re.NoError(err) + re.Equal(keyspace, loadedKeyspace) + } +} + +func TestLoadRangeKeyspaces(t *testing.T) { + re := require.New(t) + storage := NewStorageWithMemoryBackend() + + keyspaces := testKeyspaces() + for _, keyspace := range keyspaces { + re.NoError(storage.SaveKeyspace(keyspace)) + } + + // load all keyspaces. + loadedKeyspaces, err := storage.LoadRangeKeyspace(keyspaces[0].GetId(), 0) + re.NoError(err) + re.ElementsMatch(keyspaces, loadedKeyspaces) + + // load keyspaces that with id no less than second test keyspace. + loadedKeyspaces2, err := storage.LoadRangeKeyspace(keyspaces[1].GetId(), 0) + re.NoError(err) + re.ElementsMatch(keyspaces[1:], loadedKeyspaces2) + + // load keyspace with the smallest id. + loadedKeyspace3, err := storage.LoadRangeKeyspace(1, 1) + re.NoError(err) + re.ElementsMatch(keyspaces[:1], loadedKeyspace3) +} + +func testKeyspaces() []*keyspacepb.KeyspaceMeta { + now := time.Now().Unix() + return []*keyspacepb.KeyspaceMeta{ + { + Id: 500, + Name: "keyspace1", + State: keyspacepb.KeyspaceState_ENABLED, + CreatedAt: now, + StateChangedAt: now, + Config: map[string]string{ + "gc_life_time": "6000", + "gc_interval": "3000", + }, + }, + { + Id: 700, + Name: "keyspace2", + State: keyspacepb.KeyspaceState_ARCHIVED, + CreatedAt: now + 300, + StateChangedAt: now + 300, + Config: map[string]string{ + "gc_life_time": "1000", + "gc_interval": "5000", + }, + }, + { + Id: 800, + Name: "keyspace3", + State: keyspacepb.KeyspaceState_DISABLED, + CreatedAt: now + 500, + StateChangedAt: now + 500, + Config: map[string]string{ + "gc_life_time": "4000", + "gc_interval": "2000", + }, + }, + } +} diff --git a/server/storage/storage.go b/server/storage/storage.go index af65dfe4a7b..f82a6b98697 100644 --- a/server/storage/storage.go +++ b/server/storage/storage.go @@ -40,6 +40,7 @@ type Storage interface { endpoint.GCSafePointStorage endpoint.MinResolvedTSStorage endpoint.KeySpaceGCSafePointStorage + endpoint.KeyspaceStorage } // NewStorageWithMemoryBackend creates a new storage with memory backend. From 235d4635b73bc801c0d9cbea62788ce5fb6fb188 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 5 Jul 2022 13:29:32 +0800 Subject: [PATCH 02/76] storage: encode spaceID string as base 10 Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/key_path.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index 8f29a165138..43831b59a58 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -37,7 +37,7 @@ const ( keySpaceGCSafePointSuffix = "gc" keyspacePrefix = "keyspaces" keyspaceMeta = "meta" - spaceIDBase = 16 + spaceIDBase = 10 ) // AppendToRootPath appends the given key to the rootPath. From 2008600b6894ddc1883778b82d7f379922478439 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 5 Jul 2022 15:34:13 +0800 Subject: [PATCH 03/76] storage: add save/load keyspace ID Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/key_path.go | 7 ++++++- server/storage/endpoint/keyspace.go | 29 +++++++++++++++++++++++++++++ server/storage/keyspace_test.go | 23 +++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index 43831b59a58..252c0c0e20e 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -37,7 +37,7 @@ const ( keySpaceGCSafePointSuffix = "gc" keyspacePrefix = "keyspaces" keyspaceMeta = "meta" - spaceIDBase = 10 + keyspaceID = "id" ) // AppendToRootPath appends the given key to the rootPath. @@ -153,3 +153,8 @@ func KeyspaceMetaPath(spaceID uint32) string { idStr := strconv.FormatUint(uint64(spaceID), spaceIDBase) return path.Join(KeyspaceMetaPrefix(), idStr) } + +// KeyspaceIDPath returns the path to keyspace id from the given name. +func KeyspaceIDPath(name string) string { + return path.Join(keyspacePrefix, keyspaceID, name) +} diff --git a/server/storage/endpoint/keyspace.go b/server/storage/endpoint/keyspace.go index fd1b4cf5fc9..67f8e58de45 100644 --- a/server/storage/endpoint/keyspace.go +++ b/server/storage/endpoint/keyspace.go @@ -15,11 +15,15 @@ package endpoint import ( + "strconv" + "github.com/gogo/protobuf/proto" "github.com/pingcap/kvproto/pkg/keyspacepb" "go.etcd.io/etcd/clientv3" ) +const spaceIDBase = 10 + type KeyspaceStorage interface { // SaveKeyspace saves the given keyspace to the storage. SaveKeyspace(*keyspacepb.KeyspaceMeta) error @@ -27,6 +31,10 @@ type KeyspaceStorage interface { LoadKeyspace(spaceID uint32, keyspace *keyspacepb.KeyspaceMeta) (bool, error) // LoadRangeKeyspace loads no more than limit keyspaces starting at startID. LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) + // SaveKeyspaceID saves keyspace name to ID lookup information. + SaveKeyspaceID(spaceID uint32, name string) error + // LoadKeyspaceID loads keyspace ID for the given keyspace specified by name. + LoadKeyspaceID(name string) (bool, uint32, error) } var _ KeyspaceStorage = (*StorageEndpoint)(nil) @@ -65,3 +73,24 @@ func (se *StorageEndpoint) LoadRangeKeyspace(startID uint32, limit int) ([]*keys } return keyspaces, nil } + +// SaveKeyspaceID saves keyspace name to ID lookup information to storage. +func (se *StorageEndpoint) SaveKeyspaceID(spaceID uint32, name string) error { + key := KeyspaceIDPath(name) + idStr := strconv.FormatUint(uint64(spaceID), spaceIDBase) + return se.Save(key, idStr) +} + +// LoadKeyspaceID loads keyspace ID for the given keyspace name +func (se *StorageEndpoint) LoadKeyspaceID(name string) (bool, uint32, error) { + key := KeyspaceIDPath(name) + idStr, err := se.Load(key) + if err != nil || idStr == "" { + return false, 0, err + } + id64, err := strconv.ParseUint(idStr, spaceIDBase, 32) + if err != nil { + return false, 0, err + } + return true, uint32(id64), nil +} diff --git a/server/storage/keyspace_test.go b/server/storage/keyspace_test.go index a51695deb5e..32d3b77a4e0 100644 --- a/server/storage/keyspace_test.go +++ b/server/storage/keyspace_test.go @@ -66,6 +66,29 @@ func TestLoadRangeKeyspaces(t *testing.T) { re.ElementsMatch(keyspaces[:1], loadedKeyspace3) } +func TestSaveLoadKeyspaceID(t *testing.T) { + re := require.New(t) + storage := NewStorageWithMemoryBackend() + + ids := []uint32{100, 200, 300} + names := []string{"keyspace1", "keyspace2", "keyspace3"} + for i := range ids { + re.NoError(storage.SaveKeyspaceID(ids[i], names[i])) + } + + for i := range names { + success, id, err := storage.LoadKeyspaceID(names[i]) + re.NoError(err) + re.True(success) + re.Equal(ids[i], id) + } + // loading non-existing id should return false, 0, nil + success, id, err := storage.LoadKeyspaceID("non-existing") + re.NoError(err) + re.False(success) + re.Equal(uint32(0), id) +} + func testKeyspaces() []*keyspacepb.KeyspaceMeta { now := time.Now().Unix() return []*keyspacepb.KeyspaceMeta{ From 78ee5c6ad14b0d1cbe4f24f2137d6aa6f5082530 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 7 Jul 2022 15:19:21 +0800 Subject: [PATCH 04/76] server: keyspace Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 +- server/keyspace/keyspace.go | 229 ++++++++++++++++++++++++++++++++++++ server/keyspace_service.go | 164 ++++++++++++++++++++++++++ server/server.go | 5 + 5 files changed, 401 insertions(+), 3 deletions(-) create mode 100644 server/keyspace/keyspace.go create mode 100644 server/keyspace_service.go diff --git a/go.mod b/go.mod index 0e9929ea982..ddfe18d86a4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/tikv/pd go 1.16 // TODO: Remove after kvproto merge -replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220704092113-f8d176ab7cb4 +replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220707065914-b0848f38449c require ( github.com/AlekSi/gocov-xml v1.0.0 diff --git a/go.sum b/go.sum index 9e0721ce726..b1218187231 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AlekSi/gocov-xml v1.0.0 h1:4QctJBgXEkbzeKz6PJy6bt3JSPNSN4I2mITYW+eKUoQ= github.com/AlekSi/gocov-xml v1.0.0/go.mod h1:J0qYeZ6tDg4oZubW9mAAgxlqw39PDfoEkzB3HXSbEuA= -github.com/AmoebaProtozoa/kvproto v0.0.0-20220704092113-f8d176ab7cb4 h1:SNXqBJDGf3s7OrgyhUtry9vZoD4IpTI9fNKvGWqcN/s= -github.com/AmoebaProtozoa/kvproto v0.0.0-20220704092113-f8d176ab7cb4/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220707065914-b0848f38449c h1:vdEMHupO5MP7D43Ym0klJwEA8VHAx3y1790klsH2e7s= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220707065914-b0848f38449c/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go new file mode 100644 index 00000000000..051e7bc0392 --- /dev/null +++ b/server/keyspace/keyspace.go @@ -0,0 +1,229 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyspace + +import ( + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/tikv/pd/pkg/syncutil" + "github.com/tikv/pd/server/id" + "github.com/tikv/pd/server/storage/endpoint" +) + +const ( + spaceIDMin = uint32(1) // 1 is the minimum value of spaceID, 0 is reserved. + spaceIDMax = ^uint32(0) >> 8 // 16777215 (Uint24Max) is the maximum value of spaceID. +) + +var ( + errKeyspaceArchived = errors.New("Keyspace already archived") + ErrKeyspaceNotFound = errors.New("Keyspace does not exist") + ErrKeyspaceNameExists = errors.New("Keyspace name already in use") + errArchiveNotSupported = errors.New("Archiving keyspace not supported currently") +) + +// Manager manages keyspace related data. +// It validates requests and provides concurrency control. +type Manager struct { + // idLock guards keyspace name to id lookup. + idLock syncutil.Mutex + + keyspaceLock syncutil.Mutex + + // idAllocator allocates keyspace id. + idAllocator id.Allocator + // store is the storage for keyspace related information. + store endpoint.KeyspaceStorage +} + +// CreateKeyspaceRequest represents necessary arguments to create a keyspace. +type CreateKeyspaceRequest struct { + name string + initialConfig map[string]string + now time.Time +} + +// UpdateKeyspaceRequest represents necessary arguments to update keyspace. +type UpdateKeyspaceRequest struct { + Name string + // Whether to update the state of keyspace, + // if set to false, NewState and Now will be ignored. + UpdateState bool + NewState keyspacepb.KeyspaceState + Now time.Time + // New KVs to put in keyspace config. + ToPut map[string]string + // KVs to remove from keyspace config. + // Removal will be performed after put. + ToDelete []string +} + +// NewKeyspaceManager creates a Manager of keyspace related data. +func NewKeyspaceManager(store endpoint.KeyspaceStorage) *Manager { + // TODO: initialize a keyspace id allocator that start from 1 with limit + return &Manager{store: store} +} + +// allocID allocate a new keyspace id. +func (manager *Manager) allocID() (uint32, error) { + newID, err := manager.idAllocator.Alloc() + if err != nil { + return 0, err + } + return uint32(newID), nil +} + +// createNameToID create a keyspace name to ID lookup entry. +// It returns error if saving keyspace name meet error or if name has already been taken. +func (manager *Manager) createNameToID(spaceID uint32, name string) error { + manager.idLock.Lock() + defer manager.idLock.Unlock() + + nameExists, _, err := manager.store.LoadKeyspaceID(name) + if err != nil { + return err + } + if nameExists { + return ErrKeyspaceNameExists + } + return manager.store.SaveKeyspaceID(spaceID, name) +} + +// CreateKeyspace create a keyspace meta with initial config. +func (manager *Manager) CreateKeyspace(request CreateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { + // allocate new id + newID, err := manager.allocID() + if err != nil { + return nil, err + } + // bind name to that id + if err = manager.createNameToID(newID, request.name); err != nil { + return nil, err + } + // TODO: use single flight for more fine grain locking + manager.keyspaceLock.Lock() + defer manager.keyspaceLock.Unlock() + + keyspace := &keyspacepb.KeyspaceMeta{ + Id: newID, + Name: request.name, + State: keyspacepb.KeyspaceState_ENABLED, + CreatedAt: request.now.Unix(), + StateChangedAt: request.now.Unix(), + Config: request.initialConfig, + } + if err := manager.store.SaveKeyspace(keyspace); err != nil { + return nil, err + } + return keyspace, nil +} + +// loadKeyspaceID returns the id of keyspace specified by name. +// It returns error if loading met error, or if keyspace not exists. +func (manager *Manager) loadKeyspaceID(name string) (uint32, error) { + loaded, spaceID, err := manager.store.LoadKeyspaceID(name) + if err != nil { + return 0, err + } + if !loaded { + return 0, ErrKeyspaceNotFound + } + return spaceID, nil +} + +// LoadKeyspace returns the keyspace specified by name. +// It returns error if loading or unmarshalling met error or if keyspace does not exist. +func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, error) { + spaceID, err := manager.loadKeyspaceID(name) + if err != nil { + return nil, err + } + keyspace := &keyspacepb.KeyspaceMeta{} + loaded, err := manager.store.LoadKeyspace(spaceID, keyspace) + if err != nil { + return nil, err + } + if !loaded { + return nil, ErrKeyspaceNotFound + } + return keyspace, nil +} + +// UpdateKeyspace update keyspace's state and config according to request. +// It returns error if saving failed, operation not allowed, or if keyspace not exists. +func (manager *Manager) UpdateKeyspace(request UpdateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { + manager.keyspaceLock.Lock() + defer manager.keyspaceLock.Unlock() + // load keyspace by id + keyspace, err := manager.LoadKeyspace(request.Name) + if err != nil { + return nil, err + } + + // disallow modifying archived keyspace + if keyspace.State == keyspacepb.KeyspaceState_ARCHIVED { + return nil, errKeyspaceArchived + } + + // update keyspace state and config + if request.UpdateState { + if err = changeKeyspaceState(keyspace, request.NewState, request.Now); err != nil { + return nil, err + } + } + changeKeyspaceConfig(keyspace, request.ToPut, request.ToDelete) + // save keyspace + if err = manager.store.SaveKeyspace(keyspace); err != nil { + return nil, err + } + return keyspace, nil +} + +// LoadRangeKeyspace load up to limit keyspaces starting from keyspace with startID. +func (manager *Manager) LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { + return manager.store.LoadRangeKeyspace(startID, limit) +} + +// changeKeyspaceState changes the state of given keyspace meta. +// It also records the time state change happened. +func changeKeyspaceState(keyspace *keyspacepb.KeyspaceMeta, newState keyspacepb.KeyspaceState, now time.Time) error { + if keyspace.State == newState { + return nil + } + // ARCHIVED is the terminal state that cannot be changed from. + if keyspace.State == keyspacepb.KeyspaceState_ARCHIVED { + return errKeyspaceArchived + } + // TODO: support changing keyspace state to ARCHIVED and gc it. + if keyspace.State == keyspacepb.KeyspaceState_ARCHIVED { + return errArchiveNotSupported + } + + keyspace.StateChangedAt = now.Unix() + keyspace.State = newState + return nil +} + +// changeKeyspaceConfig update a keyspace's config according to toPut and toDelete. +func changeKeyspaceConfig(keyspace *keyspacepb.KeyspaceMeta, toPut map[string]string, toDelete []string) { + for k, v := range toPut { + keyspace.Config[k] = v + } + for _, key := range toDelete { + delete(keyspace.Config, key) + } +} diff --git a/server/keyspace_service.go b/server/keyspace_service.go new file mode 100644 index 00000000000..4e667b6f0a0 --- /dev/null +++ b/server/keyspace_service.go @@ -0,0 +1,164 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "github.com/gogo/protobuf/proto" + "github.com/tikv/pd/server/keyspace" + + "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/tikv/pd/server/storage/endpoint" + "go.etcd.io/etcd/clientv3" +) + +// KeyspaceServer wraps Server to provide keyspace service. +type KeyspaceServer struct { + *Server +} + +func (s *KeyspaceServer) header() *pdpb.ResponseHeader { + return &pdpb.ResponseHeader{ClusterId: s.clusterID} +} + +func (s *KeyspaceServer) errorHeader(err *pdpb.Error) *pdpb.ResponseHeader { + return &pdpb.ResponseHeader{ + ClusterId: s.clusterID, + Error: err, + } +} + +func (s *KeyspaceServer) notBootstrappedHeader() *pdpb.ResponseHeader { + return s.errorHeader(&pdpb.Error{ + Type: pdpb.ErrorType_NOT_BOOTSTRAPPED, + Message: "cluster is not bootstrapped", + }) +} + +// getErrorHeader returns corresponding ResponseHeader based on err. +func (s *KeyspaceServer) getErrorHeader(err error) *pdpb.ResponseHeader { + switch err { + case keyspace.ErrKeyspaceNameExists: + return s.errorHeader(&pdpb.Error{ + Type: pdpb.ErrorType_DUPLICATED_ENTRY, + Message: err.Error(), + }) + case keyspace.ErrKeyspaceNotFound: + return s.errorHeader(&pdpb.Error{ + Type: pdpb.ErrorType_ENTRY_NOT_FOUND, + Message: err.Error(), + }) + default: + return s.errorHeader(&pdpb.Error{ + Type: pdpb.ErrorType_UNKNOWN, + Message: err.Error(), + }) + } +} + +func (s *KeyspaceServer) UpdateKeyspaceConfig(ctx context.Context, request *keyspacepb.UpdateKeyspaceConfigRequest) (*keyspacepb.UpdateKeyspaceConfigResponse, error) { + rc := s.GetRaftCluster() + if rc == nil { + return &keyspacepb.UpdateKeyspaceConfigResponse{Header: s.notBootstrappedHeader()}, nil + } + + manager := s.keyspaceManager + updateRequest := keyspace.UpdateKeyspaceRequest{ + Name: request.Name, + UpdateState: false, + ToPut: request.Put, + ToDelete: request.Delete, + } + keyspaceMeta, err := manager.UpdateKeyspace(updateRequest) + if err != nil { + return &keyspacepb.UpdateKeyspaceConfigResponse{Header: s.getErrorHeader(err)}, err + } + + return &keyspacepb.UpdateKeyspaceConfigResponse{ + Header: s.header(), + Keyspace: keyspaceMeta, + }, nil +} + +func (s *KeyspaceServer) LoadKeyspace(ctx context.Context, request *keyspacepb.LoadKeyspaceRequest) (*keyspacepb.LoadKeyspaceResponse, error) { + rc := s.GetRaftCluster() + if rc == nil { + return &keyspacepb.LoadKeyspaceResponse{Header: s.notBootstrappedHeader()}, nil + } + + manager := s.keyspaceManager + meta, err := manager.LoadKeyspace(request.Name) + if err != nil { + return &keyspacepb.LoadKeyspaceResponse{Header: s.getErrorHeader(err)}, err + } + return &keyspacepb.LoadKeyspaceResponse{ + Header: s.header(), + Keyspace: meta, + }, nil +} + +func (s *KeyspaceServer) WatchKeyspaces(_ *keyspacepb.WatchKeyspacesRequest, stream keyspacepb.Keyspace_WatchKeyspacesServer) error { + rc := s.GetRaftCluster() + if rc == nil { + return stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.notBootstrappedHeader()}) + } + + ctx, cancel := context.WithCancel(s.Context()) + defer cancel() + + err := s.sendAllKeyspaceMeta(ctx, stream) + if err != nil { + return err + } + watchChan := s.client.Watch(ctx, endpoint.KeyspaceMetaPrefix(), clientv3.WithPrefix()) + for { + select { + case <-ctx.Done(): + return nil + case res := <-watchChan: + keyspaces := make([]*keyspacepb.KeyspaceMeta, 0, len(res.Events)) + for _, event := range res.Events { + if event.Type != clientv3.EventTypePut { + continue + } + keyspace := &keyspacepb.KeyspaceMeta{} + if err = proto.Unmarshal(event.Kv.Value, keyspace); err != nil { + return err + } + keyspaces = append(keyspaces, keyspace) + } + if len(keyspaces) > 0 { + if err = stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.header(), Keyspaces: keyspaces}); err != nil { + return err + } + } + } + } +} + +func (s *KeyspaceServer) sendAllKeyspaceMeta(ctx context.Context, stream keyspacepb.Keyspace_WatchKeyspacesServer) error { + getResp, err := s.client.Get(ctx, endpoint.KeyspaceMetaPrefix(), clientv3.WithPrefix()) + if err != nil { + return err + } + keyspaces := make([]*keyspacepb.KeyspaceMeta, getResp.Count) + for i, kv := range getResp.Kvs { + if err = proto.Unmarshal(kv.Value, keyspaces[i]); err != nil { + return err + } + } + return stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.header(), Keyspaces: keyspaces}) +} diff --git a/server/server.go b/server/server.go index 6e900580d66..e7bdb2c149d 100644 --- a/server/server.go +++ b/server/server.go @@ -35,6 +35,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/diagnosticspb" + "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" @@ -55,6 +56,7 @@ import ( "github.com/tikv/pd/server/encryptionkm" "github.com/tikv/pd/server/gc" "github.com/tikv/pd/server/id" + "github.com/tikv/pd/server/keyspace" "github.com/tikv/pd/server/member" syncer "github.com/tikv/pd/server/region_syncer" "github.com/tikv/pd/server/schedule" @@ -130,6 +132,8 @@ type Server struct { storage storage.Storage // safepoint manager gcSafePointManager *gc.SafePointManager + // keyspace manager + keyspaceManager *keyspace.Manager // for basicCluster operation. basicCluster *core.BasicCluster // for tso. @@ -276,6 +280,7 @@ func CreateServer(ctx context.Context, cfg *config.Config, serviceBuilders ...Ha } etcdCfg.ServiceRegister = func(gs *grpc.Server) { pdpb.RegisterPDServer(gs, &GrpcServer{Server: s}) + keyspacepb.RegisterKeyspaceServer(gs, &KeyspaceServer{Server: s}) diagnosticspb.RegisterDiagnosticsServer(gs, s) } s.etcdCfg = etcdCfg From 01bb8e16379e18bdc98c324d91bbdc14efaf8601 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 7 Jul 2022 15:46:47 +0800 Subject: [PATCH 05/76] make id allocator more general purporse Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/id/id.go | 27 +++++++++++++++++++-------- server/id/metrics.go | 2 -- server/server.go | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/server/id/id.go b/server/id/id.go index bca376d70b8..789e5844617 100644 --- a/server/id/id.go +++ b/server/id/id.go @@ -18,6 +18,7 @@ import ( "path" "github.com/pingcap/log" + "github.com/prometheus/client_golang/prometheus" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/etcdutil" "github.com/tikv/pd/pkg/syncutil" @@ -45,14 +46,24 @@ type allocatorImpl struct { base uint64 end uint64 - client *clientv3.Client - rootPath string - member string + client *clientv3.Client + rootPath string + allocPath string + label string + metric prometheus.Gauge + member string } // NewAllocator creates a new ID Allocator. -func NewAllocator(client *clientv3.Client, rootPath string, member string) Allocator { - return &allocatorImpl{client: client, rootPath: rootPath, member: member} +func NewAllocator(client *clientv3.Client, rootPath, allocPath, label, member string) Allocator { + return &allocatorImpl{ + client: client, + rootPath: rootPath, + allocPath: allocPath, + label: label, + metric: idGauge.WithLabelValues(label), + member: member, + } } // Alloc returns a new id. @@ -119,13 +130,13 @@ func (alloc *allocatorImpl) rebaseLocked() error { return errs.ErrEtcdTxnConflict.FastGenByArgs() } - log.Info("idAllocator allocates a new id", zap.Uint64("alloc-id", end)) - idallocGauge.Set(float64(end)) + log.Info("idAllocator allocates a new id", zap.String("label", alloc.label), zap.Uint64("alloc-id", end)) + alloc.metric.Set(float64(end)) alloc.end = end alloc.base = end - allocStep return nil } func (alloc *allocatorImpl) getAllocIDPath() string { - return path.Join(alloc.rootPath, "alloc_id") + return path.Join(alloc.rootPath, alloc.allocPath) } diff --git a/server/id/metrics.go b/server/id/metrics.go index 958582d023e..bd7e1cc32ba 100644 --- a/server/id/metrics.go +++ b/server/id/metrics.go @@ -24,8 +24,6 @@ var ( Name: "id", Help: "Record of id allocator.", }, []string{"type"}) - - idallocGauge = idGauge.WithLabelValues("idalloc") ) func init() { diff --git a/server/server.go b/server/server.go index e7bdb2c149d..28f12334902 100644 --- a/server/server.go +++ b/server/server.go @@ -386,7 +386,7 @@ func (s *Server) startServer(ctx context.Context) error { s.member.SetMemberDeployPath(s.member.ID()) s.member.SetMemberBinaryVersion(s.member.ID(), versioninfo.PDReleaseVersion) s.member.SetMemberGitHash(s.member.ID(), versioninfo.PDGitHash) - s.idAllocator = id.NewAllocator(s.client, s.rootPath, s.member.MemberValue()) + s.idAllocator = id.NewAllocator(s.client, s.rootPath, "alloc_id", "idalloc", s.member.MemberValue()) s.tsoAllocatorManager = tso.NewAllocatorManager( s.member, s.rootPath, s.cfg, func() time.Duration { return s.persistOptions.GetMaxResetTSGap() }) From 406a79fcb1d11a723ec04a59c05e2ee59a76ec76 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 7 Jul 2022 17:26:08 +0800 Subject: [PATCH 06/76] use new idAllocator Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 22 +++++++++++++++++----- server/server.go | 2 ++ server/storage/endpoint/key_path.go | 7 +++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index 051e7bc0392..9ace138bec8 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -34,6 +34,7 @@ var ( ErrKeyspaceNotFound = errors.New("Keyspace does not exist") ErrKeyspaceNameExists = errors.New("Keyspace name already in use") errArchiveNotSupported = errors.New("Archiving keyspace not supported currently") + errIDUsedUp = errors.New("Keyspace ID too large") ) // Manager manages keyspace related data. @@ -73,18 +74,29 @@ type UpdateKeyspaceRequest struct { } // NewKeyspaceManager creates a Manager of keyspace related data. -func NewKeyspaceManager(store endpoint.KeyspaceStorage) *Manager { - // TODO: initialize a keyspace id allocator that start from 1 with limit - return &Manager{store: store} +func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator) *Manager { + return &Manager{ + store: store, + idAllocator: idAllocator, + } } // allocID allocate a new keyspace id. func (manager *Manager) allocID() (uint32, error) { - newID, err := manager.idAllocator.Alloc() + id64, err := manager.idAllocator.Alloc() if err != nil { return 0, err } - return uint32(newID), nil + // id allocated id too small, keep allocating + id32 := uint32(id64) + if id32 == spaceIDMin { + return manager.allocID() + } + // if allocated id too big, return error + if id32 > spaceIDMax { + return 0, errIDUsedUp + } + return id32, nil } // createNameToID create a keyspace name to ID lookup entry. diff --git a/server/server.go b/server/server.go index 28f12334902..8cf5d2c101a 100644 --- a/server/server.go +++ b/server/server.go @@ -408,6 +408,8 @@ func (s *Server) startServer(ctx context.Context) error { defaultStorage := storage.NewStorageWithEtcdBackend(s.client, s.rootPath) s.storage = storage.NewCoreStorage(defaultStorage, regionStorage) s.gcSafePointManager = gc.NewSafePointManager(s.storage) + keyspaceIDAllocator := id.NewAllocator(s.client, s.rootPath, endpoint.KeyspaceIDAlloc(), "keyspace-idAlloc", s.member.MemberValue()) + s.keyspaceManager = keyspace.NewKeyspaceManager(s.storage, keyspaceIDAllocator) s.basicCluster = core.NewBasicCluster() s.cluster = cluster.NewRaftCluster(ctx, s.clusterID, syncer.NewRegionSyncer(s), s.client, s.httpClient) s.hbStreams = hbstream.NewHeartbeatStreams(ctx, s.clusterID, s.cluster) diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index 252c0c0e20e..831ace0d470 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -38,6 +38,7 @@ const ( keyspacePrefix = "keyspaces" keyspaceMeta = "meta" keyspaceID = "id" + keyspaceAllocID = "alloc_id" ) // AppendToRootPath appends the given key to the rootPath. @@ -158,3 +159,9 @@ func KeyspaceMetaPath(spaceID uint32) string { func KeyspaceIDPath(name string) string { return path.Join(keyspacePrefix, keyspaceID, name) } + +// KeyspaceIDAlloc returns the path of the keyspace id's persistent window boundary. +// Path: /keyspace/alloc_id +func KeyspaceIDAlloc() string { + return path.Join(keyspacePrefix, keyspaceAllocID) +} From 60b72e99624ee67372fbfed477bdd86155598e94 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 8 Jul 2022 15:59:10 +0800 Subject: [PATCH 07/76] added tests Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 59 +++++++------ server/keyspace/keyspace_test.go | 143 +++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 30 deletions(-) create mode 100644 server/keyspace/keyspace_test.go diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index 9ace138bec8..94502765b5d 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -30,21 +30,20 @@ const ( ) var ( - errKeyspaceArchived = errors.New("Keyspace already archived") - ErrKeyspaceNotFound = errors.New("Keyspace does not exist") - ErrKeyspaceNameExists = errors.New("Keyspace name already in use") - errArchiveNotSupported = errors.New("Archiving keyspace not supported currently") - errIDUsedUp = errors.New("Keyspace ID too large") + errKeyspaceArchived = errors.New("Keyspace already archived") + ErrKeyspaceNotFound = errors.New("Keyspace does not exist") + ErrKeyspaceNameExists = errors.New("Keyspace name already in use") + errIllegalID = errors.New("Cannot create keyspace with that ID") + errIllegalName = errors.New("Cannot create keyspace with that name") ) // Manager manages keyspace related data. // It validates requests and provides concurrency control. type Manager struct { - // idLock guards keyspace name to id lookup. + // idLock guards keyspace name to id lookup entries. idLock syncutil.Mutex - - keyspaceLock syncutil.Mutex - + // metaLock guards keyspace meta. + metaLock syncutil.Mutex // idAllocator allocates keyspace id. idAllocator id.Allocator // store is the storage for keyspace related information. @@ -53,9 +52,9 @@ type Manager struct { // CreateKeyspaceRequest represents necessary arguments to create a keyspace. type CreateKeyspaceRequest struct { - name string - initialConfig map[string]string - now time.Time + Name string + InitialConfig map[string]string + Now time.Time } // UpdateKeyspaceRequest represents necessary arguments to update keyspace. @@ -87,14 +86,14 @@ func (manager *Manager) allocID() (uint32, error) { if err != nil { return 0, err } - // id allocated id too small, keep allocating id32 := uint32(id64) - if id32 == spaceIDMin { + // skip id 0 + if id32 < spaceIDMin { return manager.allocID() } // if allocated id too big, return error if id32 > spaceIDMax { - return 0, errIDUsedUp + return 0, errIllegalID } return id32, nil } @@ -115,7 +114,7 @@ func (manager *Manager) createNameToID(spaceID uint32, name string) error { return manager.store.SaveKeyspaceID(spaceID, name) } -// CreateKeyspace create a keyspace meta with initial config. +// CreateKeyspace create a keyspace meta with initial config and save it to storage. func (manager *Manager) CreateKeyspace(request CreateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { // allocate new id newID, err := manager.allocID() @@ -123,20 +122,24 @@ func (manager *Manager) CreateKeyspace(request CreateKeyspaceRequest) (*keyspace return nil, err } // bind name to that id - if err = manager.createNameToID(newID, request.name); err != nil { + if err = manager.createNameToID(newID, request.Name); err != nil { return nil, err } - // TODO: use single flight for more fine grain locking - manager.keyspaceLock.Lock() - defer manager.keyspaceLock.Unlock() + + if request.Name == "" { + return nil, errIllegalName + } + + manager.metaLock.Lock() + defer manager.metaLock.Unlock() keyspace := &keyspacepb.KeyspaceMeta{ Id: newID, - Name: request.name, + Name: request.Name, State: keyspacepb.KeyspaceState_ENABLED, - CreatedAt: request.now.Unix(), - StateChangedAt: request.now.Unix(), - Config: request.initialConfig, + CreatedAt: request.Now.Unix(), + StateChangedAt: request.Now.Unix(), + Config: request.InitialConfig, } if err := manager.store.SaveKeyspace(keyspace); err != nil { return nil, err @@ -178,8 +181,8 @@ func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, err // UpdateKeyspace update keyspace's state and config according to request. // It returns error if saving failed, operation not allowed, or if keyspace not exists. func (manager *Manager) UpdateKeyspace(request UpdateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { - manager.keyspaceLock.Lock() - defer manager.keyspaceLock.Unlock() + manager.metaLock.Lock() + defer manager.metaLock.Unlock() // load keyspace by id keyspace, err := manager.LoadKeyspace(request.Name) if err != nil { @@ -220,10 +223,6 @@ func changeKeyspaceState(keyspace *keyspacepb.KeyspaceMeta, newState keyspacepb. if keyspace.State == keyspacepb.KeyspaceState_ARCHIVED { return errKeyspaceArchived } - // TODO: support changing keyspace state to ARCHIVED and gc it. - if keyspace.State == keyspacepb.KeyspaceState_ARCHIVED { - return errArchiveNotSupported - } keyspace.StateChangedAt = now.Unix() keyspace.State = newState diff --git a/server/keyspace/keyspace_test.go b/server/keyspace/keyspace_test.go new file mode 100644 index 00000000000..ec07f23235e --- /dev/null +++ b/server/keyspace/keyspace_test.go @@ -0,0 +1,143 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyspace + +import ( + "fmt" + "github.com/pingcap/kvproto/pkg/keyspacepb" + "math/rand" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/mock/mockid" + "github.com/tikv/pd/server/storage/endpoint" + "github.com/tikv/pd/server/storage/kv" +) + +func newKeyspaceManager() *Manager { + store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) + allocator := mockid.NewIDAllocator() + return NewKeyspaceManager(store, allocator) +} + +func createKeyspaceRequests(count int) []CreateKeyspaceRequest { + now := time.Now() + requests := make([]CreateKeyspaceRequest, count) + for i := 0; i < count; i++ { + requests[i] = CreateKeyspaceRequest{ + Name: fmt.Sprintf("test_keyspace%d", i), + InitialConfig: map[string]string{ + "config_entry_1": "100", + "config_entry_2": "200", + }, + Now: now, + } + } + return requests +} +func TestCreateKeyspace(t *testing.T) { + re := require.New(t) + manager := newKeyspaceManager() + requests := createKeyspaceRequests(10) + + for i, request := range requests { + expected := &keyspacepb.KeyspaceMeta{ + Id: uint32(i + 1), + Name: request.Name, + State: keyspacepb.KeyspaceState_ENABLED, + CreatedAt: request.Now.Unix(), + StateChangedAt: request.Now.Unix(), + Config: request.InitialConfig, + } + + created, err := manager.CreateKeyspace(request) + re.NoError(err) + re.Equal(expected, created) + + loaded, err := manager.LoadKeyspace(request.Name) + re.NoError(err) + re.Equal(expected, loaded) + } + + // create a keyspace with existing name must return error + _, err := manager.CreateKeyspace(requests[0]) + re.Error(err) + + // create a keyspace with empty name must return error + _, err = manager.CreateKeyspace(CreateKeyspaceRequest{Name: ""}) + re.Error(err) +} + +func TestUpdateKeyspace(t *testing.T) { + re := require.New(t) + manager := newKeyspaceManager() + requests := createKeyspaceRequests(10) + for i, createRequest := range requests { + _, err := manager.CreateKeyspace(createRequest) + re.NoError(err) + updateRequest := UpdateKeyspaceRequest{ + Name: createRequest.Name, + UpdateState: true, + NewState: keyspacepb.KeyspaceState(rand.Int31n(3)), + Now: time.Now(), + ToPut: map[string]string{ + "config_entry_1": strconv.Itoa(i), + "config_entry_2": strconv.Itoa(i), + "additional_config": strconv.Itoa(i), + }, + ToDelete: []string{"config_entry_2"}, + } + + updated, err := manager.UpdateKeyspace(updateRequest) + re.NoError(err) + + re.Equal(updateRequest.NewState, updated.State) + if updateRequest.NewState != keyspacepb.KeyspaceState_ENABLED { + // if state changed, then new state change time must be recorded + re.Equal(updateRequest.Now.Unix(), updated.StateChangedAt) + } else { + // otherwise state change time should not change + re.Equal(createRequest.Now.Unix(), updated.StateChangedAt) + } + // check for puts + re.Equal(strconv.Itoa(i), updated.Config["config_entry_1"]) + re.Equal(strconv.Itoa(i), updated.Config["additional_config"]) + // deleted key must be deleted after put + _, ok := updated.Config["config_entry_2"] + re.False(ok) + } +} + +func TestLoadRangeKeyspace(t *testing.T) { + re := require.New(t) + manager := newKeyspaceManager() + requests := createKeyspaceRequests(10) + + // force keyspace id start with 100 + for i := 0; i < 100; i++ { + _, _ = manager.idAllocator.Alloc() + } + for _, createRequest := range requests { + _, err := manager.CreateKeyspace(createRequest) + re.NoError(err) + } + + // load all 10 of them + keyspaces, err := manager.LoadRangeKeyspace(100, 10) + re.NoError(err) + re.Equal(10, len(keyspaces)) +} From e3de2be575280c0ee491c279366ce96a41b81c2c Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 11 Jul 2022 14:58:06 +0800 Subject: [PATCH 08/76] server: added concurrent update tests Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace_test.go | 150 ++++++++++++++++++++++++------- 1 file changed, 118 insertions(+), 32 deletions(-) diff --git a/server/keyspace/keyspace_test.go b/server/keyspace/keyspace_test.go index ec07f23235e..3df9893dd95 100644 --- a/server/keyspace/keyspace_test.go +++ b/server/keyspace/keyspace_test.go @@ -16,18 +16,21 @@ package keyspace import ( "fmt" - "github.com/pingcap/kvproto/pkg/keyspacepb" "math/rand" "strconv" + "sync" "testing" "time" + "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/server/storage/endpoint" "github.com/tikv/pd/server/storage/kv" ) +const testConfig = "test config" + func newKeyspaceManager() *Manager { store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) allocator := mockid.NewIDAllocator() @@ -55,22 +58,15 @@ func TestCreateKeyspace(t *testing.T) { requests := createKeyspaceRequests(10) for i, request := range requests { - expected := &keyspacepb.KeyspaceMeta{ - Id: uint32(i + 1), - Name: request.Name, - State: keyspacepb.KeyspaceState_ENABLED, - CreatedAt: request.Now.Unix(), - StateChangedAt: request.Now.Unix(), - Config: request.InitialConfig, - } - created, err := manager.CreateKeyspace(request) re.NoError(err) - re.Equal(expected, created) + re.Equal(uint32(i+1), created.Id) + matchCreateRequest(re, request, created) loaded, err := manager.LoadKeyspace(request.Name) re.NoError(err) - re.Equal(expected, loaded) + re.Equal(uint32(i+1), loaded.Id) + matchCreateRequest(re, request, loaded) } // create a keyspace with existing name must return error @@ -85,7 +81,7 @@ func TestCreateKeyspace(t *testing.T) { func TestUpdateKeyspace(t *testing.T) { re := require.New(t) manager := newKeyspaceManager() - requests := createKeyspaceRequests(10) + requests := createKeyspaceRequests(5) for i, createRequest := range requests { _, err := manager.CreateKeyspace(createRequest) re.NoError(err) @@ -104,21 +100,7 @@ func TestUpdateKeyspace(t *testing.T) { updated, err := manager.UpdateKeyspace(updateRequest) re.NoError(err) - - re.Equal(updateRequest.NewState, updated.State) - if updateRequest.NewState != keyspacepb.KeyspaceState_ENABLED { - // if state changed, then new state change time must be recorded - re.Equal(updateRequest.Now.Unix(), updated.StateChangedAt) - } else { - // otherwise state change time should not change - re.Equal(createRequest.Now.Unix(), updated.StateChangedAt) - } - // check for puts - re.Equal(strconv.Itoa(i), updated.Config["config_entry_1"]) - re.Equal(strconv.Itoa(i), updated.Config["additional_config"]) - // deleted key must be deleted after put - _, ok := updated.Config["config_entry_2"] - re.False(ok) + matchUpdateRequest(re, updateRequest, updated) } } @@ -127,8 +109,9 @@ func TestLoadRangeKeyspace(t *testing.T) { manager := newKeyspaceManager() requests := createKeyspaceRequests(10) - // force keyspace id start with 100 - for i := 0; i < 100; i++ { + startID := 100 + // force keyspace id start with startID + for i := 0; i < startID-1; i++ { _, _ = manager.idAllocator.Alloc() } for _, createRequest := range requests { @@ -136,8 +119,111 @@ func TestLoadRangeKeyspace(t *testing.T) { re.NoError(err) } - // load all 10 of them - keyspaces, err := manager.LoadRangeKeyspace(100, 10) + // load all of them + keyspaces, err := manager.LoadRangeKeyspace(0, 0) re.NoError(err) re.Equal(10, len(keyspaces)) + for i := range keyspaces { + re.Equal(uint32(startID+i), keyspaces[i].Id) + matchCreateRequest(re, requests[i], keyspaces[i]) + } + + // load first 3 + keyspaces, err = manager.LoadRangeKeyspace(0, 3) + re.NoError(err) + re.Equal(3, len(keyspaces)) + for i := range keyspaces { + re.Equal(uint32(startID+i), keyspaces[i].Id) + matchCreateRequest(re, requests[i], keyspaces[i]) + } + + // load 3 starting with startID + 5 + loadStart := startID + 5 + keyspaces, err = manager.LoadRangeKeyspace(uint32(loadStart), 3) + re.NoError(err) + re.Equal(3, len(keyspaces)) + for i := range keyspaces { + re.Equal(uint32(loadStart+i), keyspaces[i].Id) + matchCreateRequest(re, requests[i+5], keyspaces[i]) + } +} + +// TestUpdateMultipleKeyspace checks that updating multiple keyspace's config simultaneously +// will be successful. +func TestUpdateMultipleKeyspace(t *testing.T) { + re := require.New(t) + manager := newKeyspaceManager() + requests := createKeyspaceRequests(5) + for _, createRequest := range requests { + _, err := manager.CreateKeyspace(createRequest) + re.NoError(err) + } + + // concurrently update all keyspaces' testConfig sequentially. + end := 100 + wg := sync.WaitGroup{} + for _, request := range requests { + wg.Add(1) + go func(name string) { + defer wg.Done() + updateKeyspaceConfig(re, manager, name, end) + }(request.Name) + } + wg.Wait() + + // check that eventually all test keyspaces' test config reaches end + for _, request := range requests { + keyspace, err := manager.LoadKeyspace(request.Name) + re.NoError(err) + re.Equal(keyspace.Config[testConfig], strconv.Itoa(end)) + } +} + +// matchCreateRequest verifies a keyspace meta matches a create request. +func matchCreateRequest(re *require.Assertions, request CreateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { + re.Equal(request.Name, meta.Name) + re.Equal(request.Now.Unix(), meta.CreatedAt) + re.Equal(request.Now.Unix(), meta.StateChangedAt) + re.Equal(keyspacepb.KeyspaceState_ENABLED, meta.State) + re.Equal(request.InitialConfig, meta.Config) +} + +// matchUpdateRequest verifies a keyspace meta could be the immediate result of an update request. +func matchUpdateRequest(re *require.Assertions, request UpdateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { + re.Equal(request.Name, meta.Name) + if request.UpdateState { + re.Equal(request.NewState, meta.State) + // keyspace's state change time at must be less (state changed) or equal (no change) than request's. + re.GreaterOrEqual(request.Now.Unix(), meta.StateChangedAt) + } + // checkMap represent kvs to check in the meta. + checkMap := make(map[string]string) + for put, putVal := range request.ToPut { + checkMap[put] = putVal + } + for _, toDelete := range request.ToDelete { + delete(checkMap, toDelete) + // must delete key from config + _, ok := meta.Config[toDelete] + re.False(ok) + } + // check that meta contains target kvs. + for checkKey, checkValue := range checkMap { + v, ok := meta.Config[checkKey] + re.True(ok) + re.Equal(checkValue, v) + } +} + +// updateKeyspaceConfig sequentially updates given keyspace's entry. +func updateKeyspaceConfig(re *require.Assertions, manager *Manager, name string, end int) { + for i := 0; i <= end; i++ { + request := UpdateKeyspaceRequest{ + Name: name, + ToPut: map[string]string{testConfig: strconv.Itoa(i)}, + } + updated, err := manager.UpdateKeyspace(request) + re.NoError(err) + matchUpdateRequest(re, request, updated) + } } From 6782d2622246996ac6b6dbbf03d8253136b83bd8 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 11 Jul 2022 17:59:23 +0800 Subject: [PATCH 09/76] client: keyspace client Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- client/client.go | 3 + client/go.mod | 3 + client/go.sum | 4 +- client/keyspace_client.go | 149 ++++++++++++++++++++++++++++++++++++++ client/metrics.go | 4 + 5 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 client/keyspace_client.go diff --git a/client/client.go b/client/client.go index 8d9ed44c77d..51ba1144f77 100644 --- a/client/client.go +++ b/client/client.go @@ -128,6 +128,9 @@ type Client interface { WatchGlobalConfig(ctx context.Context) (chan []GlobalConfigItem, error) // UpdateOption updates the client option. UpdateOption(option DynamicOption, value interface{}) error + + // KeyspaceClient manages keyspace metadata. + KeyspaceClient // Close closes the client. Close() } diff --git a/client/go.mod b/client/go.mod index 56380b0b51c..5e51cbb3f9f 100644 --- a/client/go.mod +++ b/client/go.mod @@ -2,6 +2,9 @@ module github.com/tikv/pd/client go 1.16 +// TODO: Remove after kvproto merge +replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220707065914-b0848f38449c + require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c diff --git a/client/go.sum b/client/go.sum index 90019b9b382..990e0f27e75 100644 --- a/client/go.sum +++ b/client/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220707065914-b0848f38449c h1:vdEMHupO5MP7D43Ym0klJwEA8VHAx3y1790klsH2e7s= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220707065914-b0848f38449c/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -104,8 +106,6 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/client/keyspace_client.go b/client/keyspace_client.go new file mode 100644 index 00000000000..b2128d6f3c4 --- /dev/null +++ b/client/keyspace_client.go @@ -0,0 +1,149 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pd + +import ( + "context" + "go.uber.org/zap" + "time" + + "github.com/opentracing/opentracing-go" + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/pingcap/log" + "github.com/tikv/pd/client/grpcutil" + "google.golang.org/grpc" +) + +// KeyspaceClient manages keyspace metadata. +type KeyspaceClient interface { + // UpdateKeyspaceConfig updates target keyspace's config. + UpdateKeyspaceConfig(ctx context.Context, name string, put map[string]string, delete []string) (*keyspacepb.KeyspaceMeta, error) + // LoadKeyspace load and return target keyspace's metadata. + LoadKeyspace(ctx context.Context, name string) (*keyspacepb.KeyspaceMeta, error) + // WatchKeyspaces watches keyspace meta changes. + WatchKeyspaces(ctx context.Context) (chan []*keyspacepb.KeyspaceMeta, error) +} + +// keyspaceClient returns the KeyspaceClient from current PD leader. +func (c *client) keyspaceClient() keyspacepb.KeyspaceClient { + if cc, ok := c.clientConns.Load(c.GetLeaderAddr()); ok { + return keyspacepb.NewKeyspaceClient(cc.(*grpc.ClientConn)) + } + return nil +} + +// UpdateKeyspaceConfig updates target keyspace config and returns the updated keyspace meta. +// Note: delete will happen after put. +func (c *client) UpdateKeyspaceConfig(ctx context.Context, name string, put map[string]string, delete []string) (*keyspacepb.KeyspaceMeta, error) { + if span := opentracing.SpanFromContext(ctx); span != nil { + span = opentracing.StartSpan("keyspaceClient.UpdateKeyspaceConfig", opentracing.ChildOf(span.Context())) + defer span.Finish() + } + start := time.Now() + defer func() { cmdDurationUpdateKeyspaceConfig.Observe(time.Since(start).Seconds()) }() + ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + req := &keyspacepb.UpdateKeyspaceConfigRequest{ + Header: c.requestHeader(), + Name: name, + Put: put, + Delete: delete, + } + ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) + resp, err := c.keyspaceClient().UpdateKeyspaceConfig(ctx, req) + cancel() + + if err != nil { + cmdFailedDurationUpdateKeyspaceConfig.Observe(time.Since(start).Seconds()) + c.ScheduleCheckLeader() + return nil, err + } + + if resp.Header.GetError() != nil { + cmdFailedDurationUpdateKeyspaceConfig.Observe(time.Since(start).Seconds()) + return nil, errors.Errorf("update keyspace %s config failed: %s", name, resp.Header.GetError().String()) + } + + return resp.Keyspace, nil +} + +// LoadKeyspace loads and returns target keyspace's metadata. +func (c *client) LoadKeyspace(ctx context.Context, name string) (*keyspacepb.KeyspaceMeta, error) { + if span := opentracing.SpanFromContext(ctx); span != nil { + span = opentracing.StartSpan("keyspaceClient.LoadKeyspace", opentracing.ChildOf(span.Context())) + defer span.Finish() + } + start := time.Now() + defer func() { cmdDurationLoadKeyspace.Observe(time.Since(start).Seconds()) }() + ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + req := &keyspacepb.LoadKeyspaceRequest{ + Header: c.requestHeader(), + Name: name, + } + ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) + resp, err := c.keyspaceClient().LoadKeyspace(ctx, req) + cancel() + + if err != nil { + cmdFailedDurationLoadKeyspace.Observe(time.Since(start).Seconds()) + c.ScheduleCheckLeader() + return nil, err + } + + if resp.Header.GetError() != nil { + cmdFailedDurationLoadKeyspace.Observe(time.Since(start).Seconds()) + return nil, errors.Errorf("Load keyspace %s failed: %s", name, resp.Header.GetError().String()) + } + + return resp.Keyspace, nil +} + +// WatchKeyspaces watches keyspace meta changes. +// It returns a stream of slices of keyspace metadata. +// The first message in stream contains all current keyspaceMeta, +// all subsequent messages contains new put events for all keyspaces. +func (c *client) WatchKeyspaces(ctx context.Context) (chan []*keyspacepb.KeyspaceMeta, error) { + keyspaceWatcherChan := make(chan []*keyspacepb.KeyspaceMeta) + req := &keyspacepb.WatchKeyspacesRequest{ + Header: c.requestHeader(), + } + stream, err := c.keyspaceClient().WatchKeyspaces(ctx, req) + if err != nil { + close(keyspaceWatcherChan) + return nil, err + } + go func() { + defer func() { + if r := recover(); r != nil { + log.Error("[pd] panic in keyspace client `WatchKeyspaces`", zap.Any("error", r)) + return + } + }() + for { + select { + case <-ctx.Done(): + close(keyspaceWatcherChan) + return + default: + resp, err := stream.Recv() + if err != nil { + return + } + keyspaceWatcherChan <- resp.Keyspaces + } + } + }() + return keyspaceWatcherChan, err +} diff --git a/client/metrics.go b/client/metrics.go index a085f095406..2e3eae8bc3f 100644 --- a/client/metrics.go +++ b/client/metrics.go @@ -99,6 +99,8 @@ var ( cmdDurationGetOperator = cmdDuration.WithLabelValues("get_operator") cmdDurationSplitRegions = cmdDuration.WithLabelValues("split_regions") cmdDurationSplitAndScatterRegions = cmdDuration.WithLabelValues("split_and_scatter_regions") + cmdDurationUpdateKeyspaceConfig = cmdDuration.WithLabelValues("update_keyspace_config") + cmdDurationLoadKeyspace = cmdDuration.WithLabelValues("load_keyspace") cmdFailDurationGetRegion = cmdFailedDuration.WithLabelValues("get_region") cmdFailDurationTSO = cmdFailedDuration.WithLabelValues("tso") @@ -110,6 +112,8 @@ var ( cmdFailedDurationGetAllStores = cmdFailedDuration.WithLabelValues("get_all_stores") cmdFailedDurationUpdateGCSafePoint = cmdFailedDuration.WithLabelValues("update_gc_safe_point") cmdFailedDurationUpdateServiceGCSafePoint = cmdFailedDuration.WithLabelValues("update_service_gc_safe_point") + cmdFailedDurationUpdateKeyspaceConfig = cmdDuration.WithLabelValues("update_keyspace_config") + cmdFailedDurationLoadKeyspace = cmdDuration.WithLabelValues("load_keyspace") requestDurationTSO = requestDuration.WithLabelValues("tso") ) From 7d1fa323e3afa58f3b656f14e18807e1e788515b Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:43:07 +0800 Subject: [PATCH 10/76] server: fix updare and create request Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 4 ++-- server/keyspace/keyspace_test.go | 16 ++++++++-------- server/keyspace_service.go | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index 94502765b5d..01fc7f692eb 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -115,7 +115,7 @@ func (manager *Manager) createNameToID(spaceID uint32, name string) error { } // CreateKeyspace create a keyspace meta with initial config and save it to storage. -func (manager *Manager) CreateKeyspace(request CreateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { +func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { // allocate new id newID, err := manager.allocID() if err != nil { @@ -180,7 +180,7 @@ func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, err // UpdateKeyspace update keyspace's state and config according to request. // It returns error if saving failed, operation not allowed, or if keyspace not exists. -func (manager *Manager) UpdateKeyspace(request UpdateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { +func (manager *Manager) UpdateKeyspace(request *UpdateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { manager.metaLock.Lock() defer manager.metaLock.Unlock() // load keyspace by id diff --git a/server/keyspace/keyspace_test.go b/server/keyspace/keyspace_test.go index 3df9893dd95..a68baa8e6b6 100644 --- a/server/keyspace/keyspace_test.go +++ b/server/keyspace/keyspace_test.go @@ -37,11 +37,11 @@ func newKeyspaceManager() *Manager { return NewKeyspaceManager(store, allocator) } -func createKeyspaceRequests(count int) []CreateKeyspaceRequest { +func createKeyspaceRequests(count int) []*CreateKeyspaceRequest { now := time.Now() - requests := make([]CreateKeyspaceRequest, count) + requests := make([]*CreateKeyspaceRequest, count) for i := 0; i < count; i++ { - requests[i] = CreateKeyspaceRequest{ + requests[i] = &CreateKeyspaceRequest{ Name: fmt.Sprintf("test_keyspace%d", i), InitialConfig: map[string]string{ "config_entry_1": "100", @@ -74,7 +74,7 @@ func TestCreateKeyspace(t *testing.T) { re.Error(err) // create a keyspace with empty name must return error - _, err = manager.CreateKeyspace(CreateKeyspaceRequest{Name: ""}) + _, err = manager.CreateKeyspace(&CreateKeyspaceRequest{Name: ""}) re.Error(err) } @@ -85,7 +85,7 @@ func TestUpdateKeyspace(t *testing.T) { for i, createRequest := range requests { _, err := manager.CreateKeyspace(createRequest) re.NoError(err) - updateRequest := UpdateKeyspaceRequest{ + updateRequest := &UpdateKeyspaceRequest{ Name: createRequest.Name, UpdateState: true, NewState: keyspacepb.KeyspaceState(rand.Int31n(3)), @@ -180,7 +180,7 @@ func TestUpdateMultipleKeyspace(t *testing.T) { } // matchCreateRequest verifies a keyspace meta matches a create request. -func matchCreateRequest(re *require.Assertions, request CreateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { +func matchCreateRequest(re *require.Assertions, request *CreateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { re.Equal(request.Name, meta.Name) re.Equal(request.Now.Unix(), meta.CreatedAt) re.Equal(request.Now.Unix(), meta.StateChangedAt) @@ -189,7 +189,7 @@ func matchCreateRequest(re *require.Assertions, request CreateKeyspaceRequest, m } // matchUpdateRequest verifies a keyspace meta could be the immediate result of an update request. -func matchUpdateRequest(re *require.Assertions, request UpdateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { +func matchUpdateRequest(re *require.Assertions, request *UpdateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { re.Equal(request.Name, meta.Name) if request.UpdateState { re.Equal(request.NewState, meta.State) @@ -218,7 +218,7 @@ func matchUpdateRequest(re *require.Assertions, request UpdateKeyspaceRequest, m // updateKeyspaceConfig sequentially updates given keyspace's entry. func updateKeyspaceConfig(re *require.Assertions, manager *Manager, name string, end int) { for i := 0; i <= end; i++ { - request := UpdateKeyspaceRequest{ + request := &UpdateKeyspaceRequest{ Name: name, ToPut: map[string]string{testConfig: strconv.Itoa(i)}, } diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 4e667b6f0a0..d1d1abb9f56 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -76,7 +76,7 @@ func (s *KeyspaceServer) UpdateKeyspaceConfig(ctx context.Context, request *keys } manager := s.keyspaceManager - updateRequest := keyspace.UpdateKeyspaceRequest{ + updateRequest := &keyspace.UpdateKeyspaceRequest{ Name: request.Name, UpdateState: false, ToPut: request.Put, From d3d0dc870a99c968c15bea2bfd77b226345da5b4 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:43:07 +0800 Subject: [PATCH 11/76] server: fix updare and create request Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 4 ++-- server/keyspace/keyspace_test.go | 16 ++++++++-------- server/keyspace_service.go | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index 94502765b5d..01fc7f692eb 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -115,7 +115,7 @@ func (manager *Manager) createNameToID(spaceID uint32, name string) error { } // CreateKeyspace create a keyspace meta with initial config and save it to storage. -func (manager *Manager) CreateKeyspace(request CreateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { +func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { // allocate new id newID, err := manager.allocID() if err != nil { @@ -180,7 +180,7 @@ func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, err // UpdateKeyspace update keyspace's state and config according to request. // It returns error if saving failed, operation not allowed, or if keyspace not exists. -func (manager *Manager) UpdateKeyspace(request UpdateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { +func (manager *Manager) UpdateKeyspace(request *UpdateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { manager.metaLock.Lock() defer manager.metaLock.Unlock() // load keyspace by id diff --git a/server/keyspace/keyspace_test.go b/server/keyspace/keyspace_test.go index 3df9893dd95..a68baa8e6b6 100644 --- a/server/keyspace/keyspace_test.go +++ b/server/keyspace/keyspace_test.go @@ -37,11 +37,11 @@ func newKeyspaceManager() *Manager { return NewKeyspaceManager(store, allocator) } -func createKeyspaceRequests(count int) []CreateKeyspaceRequest { +func createKeyspaceRequests(count int) []*CreateKeyspaceRequest { now := time.Now() - requests := make([]CreateKeyspaceRequest, count) + requests := make([]*CreateKeyspaceRequest, count) for i := 0; i < count; i++ { - requests[i] = CreateKeyspaceRequest{ + requests[i] = &CreateKeyspaceRequest{ Name: fmt.Sprintf("test_keyspace%d", i), InitialConfig: map[string]string{ "config_entry_1": "100", @@ -74,7 +74,7 @@ func TestCreateKeyspace(t *testing.T) { re.Error(err) // create a keyspace with empty name must return error - _, err = manager.CreateKeyspace(CreateKeyspaceRequest{Name: ""}) + _, err = manager.CreateKeyspace(&CreateKeyspaceRequest{Name: ""}) re.Error(err) } @@ -85,7 +85,7 @@ func TestUpdateKeyspace(t *testing.T) { for i, createRequest := range requests { _, err := manager.CreateKeyspace(createRequest) re.NoError(err) - updateRequest := UpdateKeyspaceRequest{ + updateRequest := &UpdateKeyspaceRequest{ Name: createRequest.Name, UpdateState: true, NewState: keyspacepb.KeyspaceState(rand.Int31n(3)), @@ -180,7 +180,7 @@ func TestUpdateMultipleKeyspace(t *testing.T) { } // matchCreateRequest verifies a keyspace meta matches a create request. -func matchCreateRequest(re *require.Assertions, request CreateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { +func matchCreateRequest(re *require.Assertions, request *CreateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { re.Equal(request.Name, meta.Name) re.Equal(request.Now.Unix(), meta.CreatedAt) re.Equal(request.Now.Unix(), meta.StateChangedAt) @@ -189,7 +189,7 @@ func matchCreateRequest(re *require.Assertions, request CreateKeyspaceRequest, m } // matchUpdateRequest verifies a keyspace meta could be the immediate result of an update request. -func matchUpdateRequest(re *require.Assertions, request UpdateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { +func matchUpdateRequest(re *require.Assertions, request *UpdateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { re.Equal(request.Name, meta.Name) if request.UpdateState { re.Equal(request.NewState, meta.State) @@ -218,7 +218,7 @@ func matchUpdateRequest(re *require.Assertions, request UpdateKeyspaceRequest, m // updateKeyspaceConfig sequentially updates given keyspace's entry. func updateKeyspaceConfig(re *require.Assertions, manager *Manager, name string, end int) { for i := 0; i <= end; i++ { - request := UpdateKeyspaceRequest{ + request := &UpdateKeyspaceRequest{ Name: name, ToPut: map[string]string{testConfig: strconv.Itoa(i)}, } diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 4e667b6f0a0..d1d1abb9f56 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -76,7 +76,7 @@ func (s *KeyspaceServer) UpdateKeyspaceConfig(ctx context.Context, request *keys } manager := s.keyspaceManager - updateRequest := keyspace.UpdateKeyspaceRequest{ + updateRequest := &keyspace.UpdateKeyspaceRequest{ Name: request.Name, UpdateState: false, ToPut: request.Put, From db42ebfb5de03bc012da30e0ef0be940610128da Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 13 Jul 2022 12:54:02 +0800 Subject: [PATCH 12/76] storage: remove keyspace Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/keyspace.go | 8 ++++++++ server/storage/keyspace_test.go | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/server/storage/endpoint/keyspace.go b/server/storage/endpoint/keyspace.go index 67f8e58de45..f9886b5560f 100644 --- a/server/storage/endpoint/keyspace.go +++ b/server/storage/endpoint/keyspace.go @@ -29,6 +29,8 @@ type KeyspaceStorage interface { SaveKeyspace(*keyspacepb.KeyspaceMeta) error // LoadKeyspace loads keyspace specified by spaceID. LoadKeyspace(spaceID uint32, keyspace *keyspacepb.KeyspaceMeta) (bool, error) + // RemoveKeyspace removes target keyspace specified by spaceID. + RemoveKeyspace(spaceID uint32) error // LoadRangeKeyspace loads no more than limit keyspaces starting at startID. LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) // SaveKeyspaceID saves keyspace name to ID lookup information. @@ -51,6 +53,12 @@ func (se *StorageEndpoint) LoadKeyspace(spaceID uint32, keyspace *keyspacepb.Key return se.loadProto(key, keyspace) } +// RemoveKeyspace removes target keyspace specified by spaceID. +func (se *StorageEndpoint) RemoveKeyspace(spaceID uint32) error { + key := KeyspaceMetaPath(spaceID) + return se.Remove(key) +} + // LoadRangeKeyspace loads keyspaces starting at startID. // limit specifies the limit of loaded keyspaces. func (se *StorageEndpoint) LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { diff --git a/server/storage/keyspace_test.go b/server/storage/keyspace_test.go index 32d3b77a4e0..b1cfeaae167 100644 --- a/server/storage/keyspace_test.go +++ b/server/storage/keyspace_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestSaveLoadKeyspace(t *testing.T) { +func TestKeyspace(t *testing.T) { re := require.New(t) storage := NewStorageWithMemoryBackend() @@ -34,10 +34,18 @@ func TestSaveLoadKeyspace(t *testing.T) { for _, keyspace := range keyspaces { spaceID := keyspace.GetId() loadedKeyspace := &keyspacepb.KeyspaceMeta{} + // Test load keyspace. success, err := storage.LoadKeyspace(spaceID, loadedKeyspace) re.True(success) re.NoError(err) re.Equal(keyspace, loadedKeyspace) + // Test remove keyspace. + re.NoError(storage.RemoveKeyspace(spaceID)) + success, err = storage.LoadKeyspace(spaceID, loadedKeyspace) + // Loading a non-existing keyspace should be unsuccessful. + re.False(success) + // Loading a non-existing keyspace should not return error. + re.NoError(err) } } From f3e0e65eaff2c085772c9b212ac2862d765b4436 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 13 Jul 2022 14:08:27 +0800 Subject: [PATCH 13/76] server: change create keyspace order Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index 01fc7f692eb..ad87c462dec 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -116,23 +116,16 @@ func (manager *Manager) createNameToID(spaceID uint32, name string) error { // CreateKeyspace create a keyspace meta with initial config and save it to storage. func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { - // allocate new id + // Validate input. + if request.Name == "" { + return nil, errIllegalName + } + // Allocate new keyspaceID. newID, err := manager.allocID() if err != nil { return nil, err } - // bind name to that id - if err = manager.createNameToID(newID, request.Name); err != nil { - return nil, err - } - - if request.Name == "" { - return nil, errIllegalName - } - - manager.metaLock.Lock() - defer manager.metaLock.Unlock() - + // Create and save keyspace metadata. keyspace := &keyspacepb.KeyspaceMeta{ Id: newID, Name: request.Name, @@ -141,9 +134,20 @@ func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspac StateChangedAt: request.Now.Unix(), Config: request.InitialConfig, } + manager.metaLock.Lock() + defer manager.metaLock.Unlock() if err := manager.store.SaveKeyspace(keyspace); err != nil { return nil, err } + // Create name to ID entry, + // if this failed, previously stored keyspace meta should be removed. + if err = manager.createNameToID(newID, request.Name); err != nil { + if removeErr := manager.store.RemoveKeyspace(newID); removeErr != nil { + return nil, errors.Wrap(removeErr, "failed to remove keyspace meta after save spaceID failure") + } + return nil, err + } + return keyspace, nil } From b6c248d7152562a47509bb36cebeb1a8ea2b62f4 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 13 Jul 2022 17:55:17 +0800 Subject: [PATCH 14/76] API: basic implementation Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 235 ++++++++++++++++++++++++++++++ server/apiv2/router.go | 5 +- server/server.go | 5 + 3 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 server/apiv2/handlers/keyspace.go diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go new file mode 100644 index 00000000000..db6be2360cd --- /dev/null +++ b/server/apiv2/handlers/keyspace.go @@ -0,0 +1,235 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "strconv" + "time" + + "github.com/gin-gonic/gin" + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/server" + "github.com/tikv/pd/server/keyspace" +) + +func RegisterKeyspace(r *gin.RouterGroup) { + router := r.Group("keyspaces") + router.POST("", CreateKeyspace) + router.GET("", LoadAllKeyspaces) + router.GET("/:name", LoadKeyspace) + router.PATCH("/:name", UpdateKeyspace) +} + +type CreateKeyspaceParams struct { + Name string `json:"name"` + Config map[string]string `json:"config"` +} + +func CreateKeyspace(c *gin.Context) { + svr := c.MustGet("server").(*server.Server) + manager := svr.GetKeyspaceManager() + createParams := &CreateKeyspaceParams{} + err := c.BindJSON(createParams) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) + return + } + req := &keyspace.CreateKeyspaceRequest{ + Name: createParams.Name, + InitialConfig: createParams.Config, + Now: time.Now(), + } + meta, err := manager.CreateKeyspace(req) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta}) +} + +func LoadKeyspace(c *gin.Context) { + svr := c.MustGet("server").(*server.Server) + manager := svr.GetKeyspaceManager() + name := c.Param("name") + meta, err := manager.LoadKeyspace(name) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta}) +} + +// parseLoadAllQuery parses LoadAllKeyspaces' query parameters. +// page_token: +// The keyspace id of the scan start. If not set, scan from keyspace with id 1. +// It's string of spaceID of the previous scan result's last element (next_page_token). +// limit: +// The maximum number of keyspace metas to return. If not set, no limit is posed. +// Every scan scans limit + 1 keyspaces (if limit != 0), the extra scanned keyspace +// is to check if there's more, and used to set next_page_token in response. +func parseLoadAllQuery(c *gin.Context) (scanStart uint32, scanLimit int, err error) { + pageToken, set := c.GetQuery("page_token") + if !set || pageToken == "" { + // If pageToken is empty or unset, then scan from spaceID of 1. + scanStart = 1 + } else { + scanStart64, err := strconv.ParseUint(pageToken, 10, 32) + if err != nil { + return 0, 0, err + } + scanStart = uint32(scanStart64) + } + + limitStr, set := c.GetQuery("limit") + if !set || limitStr == "" || limitStr == "0" { + // If limit is unset or empty or 0, then no limit is posed for scan. + scanLimit = 0 + } else { + scanLimit64, err := strconv.ParseInt(limitStr, 10, 64) + if err != nil { + return 0, 0, err + } + // Scan an extra element for next_page_token. + scanLimit = int(scanLimit64) + 1 + } + + return scanStart, scanLimit, nil +} + +type LoadAllKeyspacesResponse struct { + Keyspaces []*KeyspaceMeta `json:"keyspaces"` + // Token that can be used to read immediate next page. + // If it's empty, then end has been reached. + NextPageToken string `json:"next_page_token"` +} + +func LoadAllKeyspaces(c *gin.Context) { + svr := c.MustGet("server").(*server.Server) + manager := svr.GetKeyspaceManager() + scanStart, scanLimit, err := parseLoadAllQuery(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) + return + } + scanned, err := manager.LoadRangeKeyspace(scanStart, scanLimit) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + resp := &LoadAllKeyspacesResponse{} + // If scanned 0 keyspaces, return result immediately. + if len(scanned) == 0 { + c.IndentedJSON(http.StatusOK, resp) + return + } + var resultKeyspaces []*KeyspaceMeta + if scanLimit == 0 || len(scanned) < scanLimit { + // No next page, all scanned are results. + resultKeyspaces = make([]*KeyspaceMeta, len(scanned)) + for i, meta := range scanned { + resultKeyspaces[i] = &KeyspaceMeta{meta} + } + } else { + // Scanned limit + 1 keyspaces, there is next page, all but last are results. + resultKeyspaces = make([]*KeyspaceMeta, len(scanned)-1) + for i := range resultKeyspaces { + resultKeyspaces[i] = &KeyspaceMeta{scanned[i]} + } + // Also set next_page_token here. + resp.NextPageToken = strconv.Itoa(int(scanned[len(scanned)-1].Id)) + } + resp.Keyspaces = resultKeyspaces + c.IndentedJSON(http.StatusOK, resp) +} + +type UpdateKeyspaceParams struct { + State string `json:"state"` + // Note: Config's values are string pointers. + // This is to differentiate between empty string "" and null value during binding. + // This is especially important when applying JSON merge patch, where null value means to remove, + // whereas empty string should simply set. + Config map[string]*string `json:"config"` +} + +func UpdateKeyspace(c *gin.Context) { + svr := c.MustGet("server").(*server.Server) + manager := svr.GetKeyspaceManager() + name := c.Param("name") + updateParams := &UpdateKeyspaceParams{} + err := c.BindJSON(updateParams) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) + return + } + req, err := getUpdateRequest(name, updateParams) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) + return + } + meta, err := manager.UpdateKeyspace(req) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta}) +} + +// getUpdateRequest converts updateKeyspaceParams to keyspace.UpdateKeyspaceRequest. +func getUpdateRequest(name string, params *UpdateKeyspaceParams) (*keyspace.UpdateKeyspaceRequest, error) { + req := &keyspace.UpdateKeyspaceRequest{ + Name: name, + } + if params.State == "" { + req.UpdateState = false + } else { + req.UpdateState = true + req.Now = time.Now() + stateVal, ok := keyspacepb.KeyspaceState_value[params.State] + if !ok { + return nil, errors.New("Illegal keyspace state") + } + req.NewState = keyspacepb.KeyspaceState(stateVal) + } + toPut := map[string]string{} + var toDelete []string + for k, v := range params.Config { + if v == nil { + toDelete = append(toDelete, k) + } else { + toPut[k] = *v + } + } + if len(toPut) > 0 { + req.ToPut = toPut + } + if len(toDelete) > 0 { + req.ToDelete = toDelete + } + return req, nil +} + +// KeyspaceMeta wraps keyspacepb.KeyspaceMeta to provide custom JSON marshal. +type KeyspaceMeta struct { + *keyspacepb.KeyspaceMeta +} + +// MarshalJSON creates custom marshal of KeyspaceMeta with the following: +// 1. Keyspace ID are removed from marshal result to avoid exposure of internal mechanics. +// 2. Keyspace State are marshaled to their corresponding name for better readability. +func (meta *KeyspaceMeta) MarshalJSON() ([]byte, error) { + return json.Marshal(&struct { + Name string `json:"name,omitempty"` + State string `json:"state,omitempty"` + CreatedAt int64 `json:"created_at,omitempty"` + StateChangedAt int64 `json:"state_changed_at,omitempty"` + Config map[string]string `json:"config,omitempty"` + }{ + meta.Name, + keyspacepb.KeyspaceState_name[int32(meta.State)], + meta.CreatedAt, + meta.StateChangedAt, + meta.Config, + }) +} diff --git a/server/apiv2/router.go b/server/apiv2/router.go index 7f551987cb6..9f106fa22cb 100644 --- a/server/apiv2/router.go +++ b/server/apiv2/router.go @@ -22,6 +22,7 @@ import ( "github.com/gin-gonic/gin" "github.com/joho/godotenv" "github.com/tikv/pd/server" + "github.com/tikv/pd/server/apiv2/handlers" "github.com/tikv/pd/server/apiv2/middlewares" ) @@ -50,7 +51,7 @@ func NewV2Handler(_ context.Context, svr *server.Server) (http.Handler, server.S c.Next() }) router.Use(middlewares.Redirector()) - _ = router.Group(apiV2Prefix) - + root := router.Group(apiV2Prefix) + handlers.RegisterKeyspace(root) return router, group, nil } diff --git a/server/server.go b/server/server.go index 8cf5d2c101a..4962077ac05 100644 --- a/server/server.go +++ b/server/server.go @@ -787,6 +787,11 @@ func (s *Server) GetTSOAllocatorManager() *tso.AllocatorManager { return s.tsoAllocatorManager } +// GetKeyspaceManager returns the keyspace manager of server. +func (s *Server) GetKeyspaceManager() *keyspace.Manager { + return s.keyspaceManager +} + // Name returns the unique etcd Name for this server in etcd cluster. func (s *Server) Name() string { return s.cfg.Name From 2311533fc6b7e1b0932a7fd03c442b184deb3e15 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 14 Jul 2022 10:57:44 +0800 Subject: [PATCH 15/76] API: added swaggo annotations Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index db6be2360cd..e0f0e055932 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -2,6 +2,7 @@ package handlers import ( "encoding/json" + "github.com/tikv/pd/server/apiv2/middlewares" "net/http" "strconv" "time" @@ -16,17 +17,28 @@ import ( func RegisterKeyspace(r *gin.RouterGroup) { router := r.Group("keyspaces") + router.Use(middlewares.BootstrapChecker()) router.POST("", CreateKeyspace) router.GET("", LoadAllKeyspaces) router.GET("/:name", LoadKeyspace) router.PATCH("/:name", UpdateKeyspace) } +// CreateKeyspaceParams represents parameters needed when creating a new keyspace. +// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. type CreateKeyspaceParams struct { Name string `json:"name"` Config map[string]string `json:"config"` } +// CreateKeyspace creates keyspace according to given input. +// @Tags keyspaces +// @Summary Create new keyspace. +// @Param body body CreateKeyspaceParams true "Create keyspace parameters" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /keyspaces [post] func CreateKeyspace(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() @@ -49,6 +61,14 @@ func CreateKeyspace(c *gin.Context) { c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta}) } +// LoadKeyspace returns target keyspace. +// @Tags keyspaces +// @Summary Get keyspace info. +// @Param name path string true "Keyspace Name" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /keyspaces/{name} [get] func LoadKeyspace(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() @@ -105,6 +125,15 @@ type LoadAllKeyspacesResponse struct { NextPageToken string `json:"next_page_token"` } +// LoadAllKeyspaces loads range of keyspaces. +// @Tags keyspaces +// @Summary list keyspaces. +// @Param page_token query string false "page token" +// @Param limit query string false "maximum number of results to return" +// @Produce json +// @Success 200 {object} LoadAllKeyspacesResponse +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /keyspaces [get] func LoadAllKeyspaces(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() @@ -144,6 +173,8 @@ func LoadAllKeyspaces(c *gin.Context) { c.IndentedJSON(http.StatusOK, resp) } +// UpdateKeyspaceParams represents parameters needed to update a keyspace. +// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. type UpdateKeyspaceParams struct { State string `json:"state"` // Note: Config's values are string pointers. @@ -153,6 +184,15 @@ type UpdateKeyspaceParams struct { Config map[string]*string `json:"config"` } +// UpdateKeyspace update keyspace. +// @Tags keyspaces +// @Summary Update keyspace metadata. +// @Param name path string true "Keyspace Name" +// @Param body body UpdateKeyspaceParams true "Update keyspace parameters" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 500 {string} string "PD server failed to proceed the request." +// Router /keyspaces/{name} [patch] func UpdateKeyspace(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() From 27ba85021f1d5e09ebf12d3ca27784d5c3437578 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 15 Jul 2022 12:44:43 +0800 Subject: [PATCH 16/76] update comments Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index e0f0e055932..4efaca8e1ac 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -1,3 +1,17 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package handlers import ( From 253b2c7019088b38fc01ea49df7f7b1824c8ce4c Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 11 Jul 2022 10:23:27 +0800 Subject: [PATCH 17/76] address comments Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 +- server/id/id.go | 47 ++++-- server/id/id_test.go | 91 +++++++++++ server/keyspace/keyspace.go | 234 ++++++++++++++++------------ server/keyspace/keyspace_test.go | 206 ++++++++++++++---------- server/keyspace_service.go | 24 ++- server/server.go | 20 ++- server/storage/endpoint/key_path.go | 24 ++- server/storage/endpoint/keyspace.go | 35 +++-- server/storage/keyspace_test.go | 39 +++-- 11 files changed, 481 insertions(+), 245 deletions(-) create mode 100644 server/id/id_test.go diff --git a/go.mod b/go.mod index ddfe18d86a4..ee5554ea1ba 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/tikv/pd go 1.16 // TODO: Remove after kvproto merge -replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220707065914-b0848f38449c +replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6 require ( github.com/AlekSi/gocov-xml v1.0.0 diff --git a/go.sum b/go.sum index b1218187231..b42257cd2b7 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AlekSi/gocov-xml v1.0.0 h1:4QctJBgXEkbzeKz6PJy6bt3JSPNSN4I2mITYW+eKUoQ= github.com/AlekSi/gocov-xml v1.0.0/go.mod h1:J0qYeZ6tDg4oZubW9mAAgxlqw39PDfoEkzB3HXSbEuA= -github.com/AmoebaProtozoa/kvproto v0.0.0-20220707065914-b0848f38449c h1:vdEMHupO5MP7D43Ym0klJwEA8VHAx3y1790klsH2e7s= -github.com/AmoebaProtozoa/kvproto v0.0.0-20220707065914-b0848f38449c/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6 h1:i9SJvU3P8t7MTEQ8zSUxDUDA3tyEf2+3I+JmWG5uw1Q= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= diff --git a/server/id/id.go b/server/id/id.go index 789e5844617..60d55f27cf1 100644 --- a/server/id/id.go +++ b/server/id/id.go @@ -38,7 +38,7 @@ type Allocator interface { Rebase() error } -const allocStep = uint64(1000) +const defaultAllocStep = uint64(1000) // allocatorImpl is used to allocate ID. type allocatorImpl struct { @@ -50,20 +50,41 @@ type allocatorImpl struct { rootPath string allocPath string label string - metric prometheus.Gauge member string + step uint64 + metrics *metrics +} + +// metrics is a collection of idAllocator's metrics. +type metrics struct { + idGauge prometheus.Gauge +} + +// AllocatorParams are parameters needed to create a new ID Allocator. +type AllocatorParams struct { + Client *clientv3.Client + RootPath string + AllocPath string // AllocPath specifies path to the persistent window boundary. + Label string // Label used to label metrics and logs. + Member string // Member value, used to check if current pd leader. + Step uint64 // Step size of each persistent window boundary increment, default 1000. } // NewAllocator creates a new ID Allocator. -func NewAllocator(client *clientv3.Client, rootPath, allocPath, label, member string) Allocator { - return &allocatorImpl{ - client: client, - rootPath: rootPath, - allocPath: allocPath, - label: label, - metric: idGauge.WithLabelValues(label), - member: member, +func NewAllocator(params *AllocatorParams) Allocator { + allocator := &allocatorImpl{ + client: params.Client, + rootPath: params.RootPath, + allocPath: params.AllocPath, + label: params.Label, + member: params.Member, + step: params.Step, + metrics: &metrics{idGauge: idGauge.WithLabelValues(params.Label)}, + } + if allocator.step == 0 { + allocator.step = defaultAllocStep } + return allocator } // Alloc returns a new id. @@ -117,7 +138,7 @@ func (alloc *allocatorImpl) rebaseLocked() error { cmp = clientv3.Compare(clientv3.Value(key), "=", string(value)) } - end += allocStep + end += alloc.step value = typeutil.Uint64ToBytes(end) txn := kv.NewSlowLogTxn(alloc.client) leaderPath := path.Join(alloc.rootPath, "leader") @@ -131,9 +152,9 @@ func (alloc *allocatorImpl) rebaseLocked() error { } log.Info("idAllocator allocates a new id", zap.String("label", alloc.label), zap.Uint64("alloc-id", end)) - alloc.metric.Set(float64(end)) + alloc.metrics.idGauge.Set(float64(end)) alloc.end = end - alloc.base = end - allocStep + alloc.base = end - alloc.step return nil } diff --git a/server/id/id_test.go b/server/id/id_test.go new file mode 100644 index 00000000000..5e6a629c687 --- /dev/null +++ b/server/id/id_test.go @@ -0,0 +1,91 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package id + +import ( + "context" + "strconv" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/etcdutil" + "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/embed" +) + +const ( + rootPath = "/pd" + leaderPath = "/pd/leader" + allocPath = "alloc_id" + label = "idalloc" + memberVal = "member" + step = uint64(500) +) + +// TestMultipleAllocator tests situation where multiple allocators that +// share rootPath and member val update their ids concurrently. +func TestMultipleAllocator(t *testing.T) { + re := require.New(t) + cfg := etcdutil.NewTestSingleConfig(t) + etcd, err := embed.StartEtcd(cfg) + defer func() { + etcd.Close() + }() + re.NoError(err) + + ep := cfg.LCUrls[0].String() + client, err := clientv3.New(clientv3.Config{ + Endpoints: []string{ep}, + }) + re.NoError(err) + + <-etcd.Server.ReadyNotify() + + // Put memberValue to leaderPath to simulate an election success. + _, err = client.Put(context.Background(), leaderPath, memberVal) + re.NoError(err) + + wg := sync.WaitGroup{} + for i := 0; i < 3; i++ { + iStr := strconv.Itoa(i) + wg.Add(1) + // All allocators share rootPath and memberVal, but they have different allocPaths, labels and steps. + allocator := NewAllocator(&AllocatorParams{ + Client: client, + RootPath: rootPath, + AllocPath: allocPath + iStr, + Label: label + iStr, + Member: memberVal, + Step: step * uint64(i), // allocator 0, 1, 2 should have step size 1000 (default), 500, 1000 respectively. + }) + go func(re *require.Assertions, allocator Allocator) { + defer wg.Done() + testAllocator(re, allocator) + }(re, allocator) + } + wg.Wait() +} + +// testAllocator sequentially updates given allocator and check if values are expected. +func testAllocator(re *require.Assertions, allocator Allocator) { + startID, err := allocator.Alloc() + re.NoError(err) + for i := startID + 1; i < startID+step*20; i++ { + id, err := allocator.Alloc() + re.NoError(err) + re.Equal(i, id) + } +} diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index ad87c462dec..3f6b29698ca 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -15,6 +15,7 @@ package keyspace import ( + "strings" "time" "github.com/pingcap/errors" @@ -27,14 +28,26 @@ import ( const ( spaceIDMin = uint32(1) // 1 is the minimum value of spaceID, 0 is reserved. spaceIDMax = ^uint32(0) >> 8 // 16777215 (Uint24Max) is the maximum value of spaceID. + // AllocStep set idAllocator's step when write persistent window boundary. + // Use a lower value for denser idAllocation in the event of frequent pd leader change. + AllocStep = uint64(100) + // AllocLabel is used to label keyspace idAllocator's metrics. + AllocLabel = "keyspace-idAlloc" + // illegalChars contains forbidden characters for keyspace name. + illegalChars = "/" ) var ( - errKeyspaceArchived = errors.New("Keyspace already archived") - ErrKeyspaceNotFound = errors.New("Keyspace does not exist") - ErrKeyspaceNameExists = errors.New("Keyspace name already in use") - errIllegalID = errors.New("Cannot create keyspace with that ID") - errIllegalName = errors.New("Cannot create keyspace with that name") + // ErrKeyspaceNotFound is used to indicate target keyspace does not exist. + ErrKeyspaceNotFound = errors.New("Keyspace does not exist") + // ErrKeyspaceExists indicates target keyspace already exists. + // Used when creating a new keyspace. + ErrKeyspaceExists = errors.New("Keyspace already exists") + errKeyspaceArchived = errors.New("Keyspace already archived") + errArchiveEnabled = errors.New("Cannot archive ENABLED keyspace") + errIllegalID = errors.New("Cannot create keyspace with that ID") + errIllegalName = errors.New("Cannot create keyspace with that name") + errIllegalOperation = errors.New("Illegal operation") ) // Manager manages keyspace related data. @@ -52,26 +65,13 @@ type Manager struct { // CreateKeyspaceRequest represents necessary arguments to create a keyspace. type CreateKeyspaceRequest struct { + // Name of the keyspace to be created. + // Using an existing name will result in error. Name string InitialConfig map[string]string Now time.Time } -// UpdateKeyspaceRequest represents necessary arguments to update keyspace. -type UpdateKeyspaceRequest struct { - Name string - // Whether to update the state of keyspace, - // if set to false, NewState and Now will be ignored. - UpdateState bool - NewState keyspacepb.KeyspaceState - Now time.Time - // New KVs to put in keyspace config. - ToPut map[string]string - // KVs to remove from keyspace config. - // Removal will be performed after put. - ToDelete []string -} - // NewKeyspaceManager creates a Manager of keyspace related data. func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator) *Manager { return &Manager{ @@ -80,51 +80,18 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator } } -// allocID allocate a new keyspace id. -func (manager *Manager) allocID() (uint32, error) { - id64, err := manager.idAllocator.Alloc() - if err != nil { - return 0, err - } - id32 := uint32(id64) - // skip id 0 - if id32 < spaceIDMin { - return manager.allocID() - } - // if allocated id too big, return error - if id32 > spaceIDMax { - return 0, errIllegalID - } - return id32, nil -} - -// createNameToID create a keyspace name to ID lookup entry. -// It returns error if saving keyspace name meet error or if name has already been taken. -func (manager *Manager) createNameToID(spaceID uint32, name string) error { - manager.idLock.Lock() - defer manager.idLock.Unlock() - - nameExists, _, err := manager.store.LoadKeyspaceID(name) - if err != nil { - return err - } - if nameExists { - return ErrKeyspaceNameExists - } - return manager.store.SaveKeyspaceID(spaceID, name) -} - // CreateKeyspace create a keyspace meta with initial config and save it to storage. func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { - // Validate input. - if request.Name == "" { - return nil, errIllegalName + // Validate purposed name's legality. + if err := validateName(request.Name); err != nil { + return nil, err } // Allocate new keyspaceID. newID, err := manager.allocID() if err != nil { return nil, err } + // TODO: Enable Transaction at storage layer to save MetaData and NameToID in a single transaction. // Create and save keyspace metadata. keyspace := &keyspacepb.KeyspaceMeta{ Id: newID, @@ -136,6 +103,15 @@ func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspac } manager.metaLock.Lock() defer manager.metaLock.Unlock() + // Check if keyspace with that id already exists. + idExists, err := manager.store.LoadKeyspace(newID, &keyspacepb.KeyspaceMeta{}) + if err != nil { + return nil, err + } + if idExists { + return nil, ErrKeyspaceExists + } + // Save keyspace meta before saving id. if err := manager.store.SaveKeyspace(keyspace); err != nil { return nil, err } @@ -151,28 +127,20 @@ func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspac return keyspace, nil } -// loadKeyspaceID returns the id of keyspace specified by name. -// It returns error if loading met error, or if keyspace not exists. -func (manager *Manager) loadKeyspaceID(name string) (uint32, error) { - loaded, spaceID, err := manager.store.LoadKeyspaceID(name) - if err != nil { - return 0, err - } - if !loaded { - return 0, ErrKeyspaceNotFound - } - return spaceID, nil -} - // LoadKeyspace returns the keyspace specified by name. // It returns error if loading or unmarshalling met error or if keyspace does not exist. func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, error) { - spaceID, err := manager.loadKeyspaceID(name) + // First get keyspace ID from the name given. + loaded, spaceID, err := manager.store.LoadKeyspaceIDByName(name) if err != nil { return nil, err } + if !loaded { + return nil, ErrKeyspaceNotFound + } + // Load the keyspace with target ID. keyspace := &keyspacepb.KeyspaceMeta{} - loaded, err := manager.store.LoadKeyspace(spaceID, keyspace) + loaded, err = manager.store.LoadKeyspace(spaceID, keyspace) if err != nil { return nil, err } @@ -182,30 +150,64 @@ func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, err return keyspace, nil } -// UpdateKeyspace update keyspace's state and config according to request. +// UpdateKeyspaceConfig apply mutations to target keyspace in order. // It returns error if saving failed, operation not allowed, or if keyspace not exists. -func (manager *Manager) UpdateKeyspace(request *UpdateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { +func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*keyspacepb.Mutation) (*keyspacepb.KeyspaceMeta, error) { manager.metaLock.Lock() defer manager.metaLock.Unlock() - // load keyspace by id - keyspace, err := manager.LoadKeyspace(request.Name) + // Load keyspace by name. + keyspace, err := manager.LoadKeyspace(name) if err != nil { return nil, err } - - // disallow modifying archived keyspace + // Changing ARCHIVED keyspace's config is not allowed. if keyspace.State == keyspacepb.KeyspaceState_ARCHIVED { return nil, errKeyspaceArchived } - - // update keyspace state and config - if request.UpdateState { - if err = changeKeyspaceState(keyspace, request.NewState, request.Now); err != nil { - return nil, err + // Update keyspace config according to mutations. + for _, mutation := range mutations { + switch mutation.Op { + case keyspacepb.Op_PUT: + keyspace.Config[string(mutation.Key)] = string(mutation.Value) + case keyspacepb.Op_DEL: + delete(keyspace.Config, string(mutation.Key)) + default: + return nil, errIllegalOperation } } - changeKeyspaceConfig(keyspace, request.ToPut, request.ToDelete) - // save keyspace + // Save the updated keyspace. + if err = manager.store.SaveKeyspace(keyspace); err != nil { + return nil, err + } + return keyspace, nil +} + +// UpdateKeyspaceState updates target keyspace to the given state if it's not already in that state. +// It returns error if saving failed, operation not allowed, or if keyspace not exists. +func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.KeyspaceState, now time.Time) (*keyspacepb.KeyspaceMeta, error) { + manager.metaLock.Lock() + defer manager.metaLock.Unlock() + // Load keyspace by name. + keyspace, err := manager.LoadKeyspace(name) + if err != nil { + return nil, err + } + // If keyspace is already in target state, then nothing needs to be change. + if keyspace.State == newState { + return keyspace, nil + } + // ARCHIVED is the terminal state that cannot be changed from. + if keyspace.State == keyspacepb.KeyspaceState_ARCHIVED { + return nil, errKeyspaceArchived + } + // Archiving an enabled keyspace directly is not allowed. + if keyspace.State == keyspacepb.KeyspaceState_ENABLED && newState == keyspacepb.KeyspaceState_ARCHIVED { + return nil, errArchiveEnabled + } + // Change keyspace state and record change time. + keyspace.StateChangedAt = now.Unix() + keyspace.State = newState + // Save the updated keyspace. if err = manager.store.SaveKeyspace(keyspace); err != nil { return nil, err } @@ -217,28 +219,58 @@ func (manager *Manager) LoadRangeKeyspace(startID uint32, limit int) ([]*keyspac return manager.store.LoadRangeKeyspace(startID, limit) } -// changeKeyspaceState changes the state of given keyspace meta. -// It also records the time state change happened. -func changeKeyspaceState(keyspace *keyspacepb.KeyspaceMeta, newState keyspacepb.KeyspaceState, now time.Time) error { - if keyspace.State == newState { - return nil +// allocID allocate a new keyspace id. +func (manager *Manager) allocID() (uint32, error) { + id64, err := manager.idAllocator.Alloc() + if err != nil { + return 0, err } - // ARCHIVED is the terminal state that cannot be changed from. - if keyspace.State == keyspacepb.KeyspaceState_ARCHIVED { - return errKeyspaceArchived + id32 := uint32(id64) + // If obtained id is too small, re-allocate a higher ID. + if id32 < spaceIDMin { + return manager.allocID() } + if err = validateID(id32); err != nil { + return 0, err + } + return id32, nil +} - keyspace.StateChangedAt = now.Unix() - keyspace.State = newState +// validateID check if keyspace falls within the acceptable range. +// It throws errIllegalID when input id is our of range. +func validateID(spaceID uint32) error { + if spaceID < spaceIDMin || spaceID > spaceIDMax { + return errIllegalID + } + return nil +} + +// validateName check if name contains illegal character. +// It throws errIllegalName when name contains illegal character. +func validateName(name string) error { + // Name should not be empty. + if name == "" { + return errIllegalName + } + // Name should not contain any illegal character. + if strings.ContainsAny(name, illegalChars) { + return errIllegalName + } return nil } -// changeKeyspaceConfig update a keyspace's config according to toPut and toDelete. -func changeKeyspaceConfig(keyspace *keyspacepb.KeyspaceMeta, toPut map[string]string, toDelete []string) { - for k, v := range toPut { - keyspace.Config[k] = v +// createNameToID create a keyspace name to ID lookup entry. +// It returns error if saving keyspace name meet error or if name has already been taken. +func (manager *Manager) createNameToID(spaceID uint32, name string) error { + manager.idLock.Lock() + defer manager.idLock.Unlock() + + nameExists, _, err := manager.store.LoadKeyspaceIDByName(name) + if err != nil { + return err } - for _, key := range toDelete { - delete(keyspace.Config, key) + if nameExists { + return ErrKeyspaceExists } + return manager.store.SaveKeyspaceIDByName(spaceID, name) } diff --git a/server/keyspace/keyspace_test.go b/server/keyspace/keyspace_test.go index a68baa8e6b6..ec5d2efbba7 100644 --- a/server/keyspace/keyspace_test.go +++ b/server/keyspace/keyspace_test.go @@ -16,7 +16,6 @@ package keyspace import ( "fmt" - "math/rand" "strconv" "sync" "testing" @@ -29,7 +28,11 @@ import ( "github.com/tikv/pd/server/storage/kv" ) -const testConfig = "test config" +const ( + testConfig = "test config" + testConfig1 = "config_entry_1" + testConfig2 = "config_entry_2" +) func newKeyspaceManager() *Manager { store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) @@ -37,114 +40,160 @@ func newKeyspaceManager() *Manager { return NewKeyspaceManager(store, allocator) } -func createKeyspaceRequests(count int) []*CreateKeyspaceRequest { +func makeCreateKeyspaceRequests(count int) []*CreateKeyspaceRequest { now := time.Now() requests := make([]*CreateKeyspaceRequest, count) for i := 0; i < count; i++ { requests[i] = &CreateKeyspaceRequest{ Name: fmt.Sprintf("test_keyspace%d", i), InitialConfig: map[string]string{ - "config_entry_1": "100", - "config_entry_2": "200", + testConfig1: "100", + testConfig2: "200", }, Now: now, } } return requests } + func TestCreateKeyspace(t *testing.T) { re := require.New(t) manager := newKeyspaceManager() - requests := createKeyspaceRequests(10) + requests := makeCreateKeyspaceRequests(10) for i, request := range requests { created, err := manager.CreateKeyspace(request) re.NoError(err) re.Equal(uint32(i+1), created.Id) - matchCreateRequest(re, request, created) + checkCreateRequest(re, request, created) loaded, err := manager.LoadKeyspace(request.Name) re.NoError(err) re.Equal(uint32(i+1), loaded.Id) - matchCreateRequest(re, request, loaded) + checkCreateRequest(re, request, loaded) } - // create a keyspace with existing name must return error + // Create a keyspace with existing name must return error. _, err := manager.CreateKeyspace(requests[0]) re.Error(err) - // create a keyspace with empty name must return error + // Create a keyspace with empty name must return error. _, err = manager.CreateKeyspace(&CreateKeyspaceRequest{Name: ""}) re.Error(err) } -func TestUpdateKeyspace(t *testing.T) { +func makeMutations() []*keyspacepb.Mutation { + return []*keyspacepb.Mutation{ + { + Op: keyspacepb.Op_PUT, + Key: []byte(testConfig1), + Value: []byte("new val"), + }, + { + Op: keyspacepb.Op_PUT, + Key: []byte("new config"), + Value: []byte("new val"), + }, + { + Op: keyspacepb.Op_DEL, + Key: []byte(testConfig2), + }, + } +} + +func TestUpdateKeyspaceConfig(t *testing.T) { re := require.New(t) manager := newKeyspaceManager() - requests := createKeyspaceRequests(5) - for i, createRequest := range requests { + requests := makeCreateKeyspaceRequests(5) + for _, createRequest := range requests { _, err := manager.CreateKeyspace(createRequest) re.NoError(err) - updateRequest := &UpdateKeyspaceRequest{ - Name: createRequest.Name, - UpdateState: true, - NewState: keyspacepb.KeyspaceState(rand.Int31n(3)), - Now: time.Now(), - ToPut: map[string]string{ - "config_entry_1": strconv.Itoa(i), - "config_entry_2": strconv.Itoa(i), - "additional_config": strconv.Itoa(i), - }, - ToDelete: []string{"config_entry_2"}, - } + mutations := makeMutations() + updated, err := manager.UpdateKeyspaceConfig(createRequest.Name, mutations) + re.NoError(err) + checkMutations(re, createRequest.InitialConfig, updated.Config, mutations) + // Changing config of a ARCHIVED keyspace is not allowed. + _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_DISABLED, time.Now()) + re.NoError(err) + _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ARCHIVED, time.Now()) + re.NoError(err) + _, err = manager.UpdateKeyspaceConfig(createRequest.Name, mutations) + re.Error(err) + } +} + +func TestUpdateKeyspaceState(t *testing.T) { + re := require.New(t) + manager := newKeyspaceManager() + requests := makeCreateKeyspaceRequests(5) + for _, createRequest := range requests { + _, err := manager.CreateKeyspace(createRequest) + re.NoError(err) + oldTime := time.Now() + // Archiving an ENABLED keyspace is not allowed. + _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ARCHIVED, oldTime) + re.Error(err) + // Disabling an ENABLED keyspace is allowed. Should update StateChangedAt. + updated, err := manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_DISABLED, oldTime) + re.NoError(err) + re.Equal(updated.State, keyspacepb.KeyspaceState_DISABLED) + re.Equal(updated.StateChangedAt, oldTime.Unix()) - updated, err := manager.UpdateKeyspace(updateRequest) + newTime := time.Now() + // Disabling an DISABLED keyspace is allowed. Should NOT update StateChangedAt. + updated, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_DISABLED, newTime) + re.NoError(err) + re.Equal(updated.State, keyspacepb.KeyspaceState_DISABLED) + re.Equal(updated.StateChangedAt, oldTime.Unix()) + // Archiving a DISABLED keyspace is allowed. Should update StateChangeAt. + updated, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ARCHIVED, newTime) re.NoError(err) - matchUpdateRequest(re, updateRequest, updated) + re.Equal(updated.State, keyspacepb.KeyspaceState_ARCHIVED) + re.Equal(updated.StateChangedAt, newTime.Unix()) + // Changing state of an ARCHIVED keyspace is not allowed. + _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ENABLED, newTime) + re.Error(err) } } func TestLoadRangeKeyspace(t *testing.T) { re := require.New(t) manager := newKeyspaceManager() - requests := createKeyspaceRequests(10) + // Test with 100 keyspaces. + total := 100 + requests := makeCreateKeyspaceRequests(total) - startID := 100 - // force keyspace id start with startID - for i := 0; i < startID-1; i++ { - _, _ = manager.idAllocator.Alloc() - } for _, createRequest := range requests { _, err := manager.CreateKeyspace(createRequest) re.NoError(err) } - // load all of them + // Load all keyspaces. keyspaces, err := manager.LoadRangeKeyspace(0, 0) re.NoError(err) - re.Equal(10, len(keyspaces)) + re.Equal(total, len(keyspaces)) for i := range keyspaces { - re.Equal(uint32(startID+i), keyspaces[i].Id) - matchCreateRequest(re, requests[i], keyspaces[i]) + re.Equal(uint32(i+1), keyspaces[i].Id) + checkCreateRequest(re, requests[i], keyspaces[i]) } - // load first 3 - keyspaces, err = manager.LoadRangeKeyspace(0, 3) + // Load first 50 keyspaces. + keyspaces, err = manager.LoadRangeKeyspace(0, 50) re.NoError(err) - re.Equal(3, len(keyspaces)) + re.Equal(50, len(keyspaces)) for i := range keyspaces { - re.Equal(uint32(startID+i), keyspaces[i].Id) - matchCreateRequest(re, requests[i], keyspaces[i]) + re.Equal(uint32(i+1), keyspaces[i].Id) + checkCreateRequest(re, requests[i], keyspaces[i]) } - // load 3 starting with startID + 5 - loadStart := startID + 5 - keyspaces, err = manager.LoadRangeKeyspace(uint32(loadStart), 3) + // Load 20 keyspaces starting from keyspace with id 33. + loadStart := 33 + keyspaces, err = manager.LoadRangeKeyspace(uint32(loadStart), 20) re.NoError(err) - re.Equal(3, len(keyspaces)) + re.Equal(20, len(keyspaces)) for i := range keyspaces { re.Equal(uint32(loadStart+i), keyspaces[i].Id) - matchCreateRequest(re, requests[i+5], keyspaces[i]) + checkCreateRequest(re, requests[i+loadStart-1], keyspaces[i]) } } @@ -153,13 +202,13 @@ func TestLoadRangeKeyspace(t *testing.T) { func TestUpdateMultipleKeyspace(t *testing.T) { re := require.New(t) manager := newKeyspaceManager() - requests := createKeyspaceRequests(5) + requests := makeCreateKeyspaceRequests(50) for _, createRequest := range requests { _, err := manager.CreateKeyspace(createRequest) re.NoError(err) } - // concurrently update all keyspaces' testConfig sequentially. + // Concurrently update all keyspaces' testConfig sequentially. end := 100 wg := sync.WaitGroup{} for _, request := range requests { @@ -171,7 +220,7 @@ func TestUpdateMultipleKeyspace(t *testing.T) { } wg.Wait() - // check that eventually all test keyspaces' test config reaches end + // Check that eventually all test keyspaces' test config reaches end for _, request := range requests { keyspace, err := manager.LoadKeyspace(request.Name) re.NoError(err) @@ -179,8 +228,8 @@ func TestUpdateMultipleKeyspace(t *testing.T) { } } -// matchCreateRequest verifies a keyspace meta matches a create request. -func matchCreateRequest(re *require.Assertions, request *CreateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { +// checkCreateRequest verifies a keyspace meta matches a create request. +func checkCreateRequest(re *require.Assertions, request *CreateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { re.Equal(request.Name, meta.Name) re.Equal(request.Now.Unix(), meta.CreatedAt) re.Equal(request.Now.Unix(), meta.StateChangedAt) @@ -188,42 +237,39 @@ func matchCreateRequest(re *require.Assertions, request *CreateKeyspaceRequest, re.Equal(request.InitialConfig, meta.Config) } -// matchUpdateRequest verifies a keyspace meta could be the immediate result of an update request. -func matchUpdateRequest(re *require.Assertions, request *UpdateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { - re.Equal(request.Name, meta.Name) - if request.UpdateState { - re.Equal(request.NewState, meta.State) - // keyspace's state change time at must be less (state changed) or equal (no change) than request's. - re.GreaterOrEqual(request.Now.Unix(), meta.StateChangedAt) - } - // checkMap represent kvs to check in the meta. - checkMap := make(map[string]string) - for put, putVal := range request.ToPut { - checkMap[put] = putVal - } - for _, toDelete := range request.ToDelete { - delete(checkMap, toDelete) - // must delete key from config - _, ok := meta.Config[toDelete] - re.False(ok) +// checkMutations verifies that performing mutations on old config would result in new config. +func checkMutations(re *require.Assertions, oldConfig, newConfig map[string]string, mutations []*keyspacepb.Mutation) { + // Copy oldConfig to expected to avoid modifying its content. + expected := map[string]string{} + for k, v := range oldConfig { + expected[k] = v } - // check that meta contains target kvs. - for checkKey, checkValue := range checkMap { - v, ok := meta.Config[checkKey] - re.True(ok) - re.Equal(checkValue, v) + for _, mutation := range mutations { + switch mutation.Op { + case keyspacepb.Op_PUT: + expected[string(mutation.Key)] = string(mutation.Value) + case keyspacepb.Op_DEL: + delete(expected, string(mutation.Key)) + } } + re.Equal(expected, newConfig) } // updateKeyspaceConfig sequentially updates given keyspace's entry. func updateKeyspaceConfig(re *require.Assertions, manager *Manager, name string, end int) { + oldMeta, err := manager.LoadKeyspace(name) + re.NoError(err) for i := 0; i <= end; i++ { - request := &UpdateKeyspaceRequest{ - Name: name, - ToPut: map[string]string{testConfig: strconv.Itoa(i)}, + mutations := []*keyspacepb.Mutation{ + { + Op: keyspacepb.Op_PUT, + Key: []byte(testConfig), + Value: []byte(strconv.Itoa(i)), + }, } - updated, err := manager.UpdateKeyspace(request) + updatedMeta, err := manager.UpdateKeyspaceConfig(name, mutations) re.NoError(err) - matchUpdateRequest(re, request, updated) + checkMutations(re, oldMeta.Config, updatedMeta.Config, mutations) + oldMeta = updatedMeta } } diff --git a/server/keyspace_service.go b/server/keyspace_service.go index d1d1abb9f56..707148ac84c 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -51,7 +51,7 @@ func (s *KeyspaceServer) notBootstrappedHeader() *pdpb.ResponseHeader { // getErrorHeader returns corresponding ResponseHeader based on err. func (s *KeyspaceServer) getErrorHeader(err error) *pdpb.ResponseHeader { switch err { - case keyspace.ErrKeyspaceNameExists: + case keyspace.ErrKeyspaceExists: return s.errorHeader(&pdpb.Error{ Type: pdpb.ErrorType_DUPLICATED_ENTRY, Message: err.Error(), @@ -76,20 +76,14 @@ func (s *KeyspaceServer) UpdateKeyspaceConfig(ctx context.Context, request *keys } manager := s.keyspaceManager - updateRequest := &keyspace.UpdateKeyspaceRequest{ - Name: request.Name, - UpdateState: false, - ToPut: request.Put, - ToDelete: request.Delete, - } - keyspaceMeta, err := manager.UpdateKeyspace(updateRequest) + updatedMeta, err := manager.UpdateKeyspaceConfig(request.Name, request.Mutations) if err != nil { return &keyspacepb.UpdateKeyspaceConfigResponse{Header: s.getErrorHeader(err)}, err } return &keyspacepb.UpdateKeyspaceConfigResponse{ Header: s.header(), - Keyspace: keyspaceMeta, + Keyspace: updatedMeta, }, nil } @@ -134,11 +128,11 @@ func (s *KeyspaceServer) WatchKeyspaces(_ *keyspacepb.WatchKeyspacesRequest, str if event.Type != clientv3.EventTypePut { continue } - keyspace := &keyspacepb.KeyspaceMeta{} - if err = proto.Unmarshal(event.Kv.Value, keyspace); err != nil { + meta := &keyspacepb.KeyspaceMeta{} + if err = proto.Unmarshal(event.Kv.Value, meta); err != nil { return err } - keyspaces = append(keyspaces, keyspace) + keyspaces = append(keyspaces, meta) } if len(keyspaces) > 0 { if err = stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.header(), Keyspaces: keyspaces}); err != nil { @@ -154,11 +148,11 @@ func (s *KeyspaceServer) sendAllKeyspaceMeta(ctx context.Context, stream keyspac if err != nil { return err } - keyspaces := make([]*keyspacepb.KeyspaceMeta, getResp.Count) + metas := make([]*keyspacepb.KeyspaceMeta, getResp.Count) for i, kv := range getResp.Kvs { - if err = proto.Unmarshal(kv.Value, keyspaces[i]); err != nil { + if err = proto.Unmarshal(kv.Value, metas[i]); err != nil { return err } } - return stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.header(), Keyspaces: keyspaces}) + return stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.header(), Keyspaces: metas}) } diff --git a/server/server.go b/server/server.go index 8cf5d2c101a..b9e19a7033a 100644 --- a/server/server.go +++ b/server/server.go @@ -83,6 +83,9 @@ const ( pdRootPath = "/pd" pdAPIPrefix = "/pd/" pdClusterIDPath = "/pd/cluster_id" + // idAllocPath for idAllocator to save persistent window's end. + idAllocPath = "alloc_id" + idAllocLabel = "idalloc" ) // EtcdStartTimeout the timeout of the startup etcd. @@ -386,7 +389,13 @@ func (s *Server) startServer(ctx context.Context) error { s.member.SetMemberDeployPath(s.member.ID()) s.member.SetMemberBinaryVersion(s.member.ID(), versioninfo.PDReleaseVersion) s.member.SetMemberGitHash(s.member.ID(), versioninfo.PDGitHash) - s.idAllocator = id.NewAllocator(s.client, s.rootPath, "alloc_id", "idalloc", s.member.MemberValue()) + s.idAllocator = id.NewAllocator(&id.AllocatorParams{ + Client: s.client, + RootPath: s.rootPath, + AllocPath: idAllocPath, + Label: idAllocLabel, + Member: s.member.MemberValue(), + }) s.tsoAllocatorManager = tso.NewAllocatorManager( s.member, s.rootPath, s.cfg, func() time.Duration { return s.persistOptions.GetMaxResetTSGap() }) @@ -408,7 +417,14 @@ func (s *Server) startServer(ctx context.Context) error { defaultStorage := storage.NewStorageWithEtcdBackend(s.client, s.rootPath) s.storage = storage.NewCoreStorage(defaultStorage, regionStorage) s.gcSafePointManager = gc.NewSafePointManager(s.storage) - keyspaceIDAllocator := id.NewAllocator(s.client, s.rootPath, endpoint.KeyspaceIDAlloc(), "keyspace-idAlloc", s.member.MemberValue()) + keyspaceIDAllocator := id.NewAllocator(&id.AllocatorParams{ + Client: s.client, + RootPath: s.rootPath, + AllocPath: endpoint.KeyspaceIDAlloc(), + Label: keyspace.AllocLabel, + Member: s.member.MemberValue(), + Step: keyspace.AllocStep, + }) s.keyspaceManager = keyspace.NewKeyspaceManager(s.storage, keyspaceIDAllocator) s.basicCluster = core.NewBasicCluster() s.cluster = cluster.NewRaftCluster(ctx, s.clusterID, syncer.NewRegionSyncer(s), s.client, s.httpClient) diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index 831ace0d470..a2ece89e0c8 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -18,6 +18,7 @@ import ( "fmt" "path" "strconv" + "strings" ) const ( @@ -36,8 +37,8 @@ const ( keySpaceSafePointPrefix = "key_space/gc_safepoint" keySpaceGCSafePointSuffix = "gc" keyspacePrefix = "keyspaces" - keyspaceMeta = "meta" - keyspaceID = "id" + keyspaceMetaInfix = "meta" + keyspaceIDInfix = "id" keyspaceAllocID = "alloc_id" ) @@ -145,23 +146,32 @@ func KeySpaceGCSafePointSuffix() string { // KeyspaceMetaPrefix returns the prefix of keyspaces' metadata. // Prefix: keyspaces/meta/ func KeyspaceMetaPrefix() string { - return path.Join(keyspacePrefix, keyspaceMeta) + "/" + return path.Join(keyspacePrefix, keyspaceMetaInfix) + "/" } // KeyspaceMetaPath returns the path to the given keyspace's metadata. -// Path: /keyspaces/meta/{space_id} +// Path: keyspaces/meta/{space_id} func KeyspaceMetaPath(spaceID uint32) string { - idStr := strconv.FormatUint(uint64(spaceID), spaceIDBase) + idStr := encodeKeyspaceID(spaceID) return path.Join(KeyspaceMetaPrefix(), idStr) } // KeyspaceIDPath returns the path to keyspace id from the given name. +// Path: keyspaces/id/{name} func KeyspaceIDPath(name string) string { - return path.Join(keyspacePrefix, keyspaceID, name) + return path.Join(keyspacePrefix, keyspaceIDInfix, name) } // KeyspaceIDAlloc returns the path of the keyspace id's persistent window boundary. -// Path: /keyspace/alloc_id +// Path: keyspace/alloc_id func KeyspaceIDAlloc() string { return path.Join(keyspacePrefix, keyspaceAllocID) } + +// encodeKeyspaceID from uint32 to string. +// It adds extra padding to make encoded ID ordered. +// Encoded ID can be decoded directly with strconv.ParseUint. +func encodeKeyspaceID(spaceID uint32) string { + idStr := strconv.FormatUint(uint64(spaceID), spaceIDBase) + return strings.Repeat("0", spaceIDStrLen-len(idStr)) + idStr +} diff --git a/server/storage/endpoint/keyspace.go b/server/storage/endpoint/keyspace.go index f9886b5560f..357346e867a 100644 --- a/server/storage/endpoint/keyspace.go +++ b/server/storage/endpoint/keyspace.go @@ -22,7 +22,18 @@ import ( "go.etcd.io/etcd/clientv3" ) -const spaceIDBase = 10 +const ( + // spaceIDBase is base used to encode/decode spaceID. + // It's set to 10 for better readability. + spaceIDBase = 10 + // spaceIDBitSizeMax is the max bitSize of spaceID. + // It's currently set to 24 (3bytes). + spaceIDBitSizeMax = 24 + // spaceIDStrLen is the length of spaceID when written in spaceIDBase. + // It's currently set to 8 (base10 of uint24max is 16777215) + // It's used to pad all spaceIDs in etcd path to the same length. + spaceIDStrLen = 8 +) type KeyspaceStorage interface { // SaveKeyspace saves the given keyspace to the storage. @@ -33,10 +44,13 @@ type KeyspaceStorage interface { RemoveKeyspace(spaceID uint32) error // LoadRangeKeyspace loads no more than limit keyspaces starting at startID. LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) - // SaveKeyspaceID saves keyspace name to ID lookup information. - SaveKeyspaceID(spaceID uint32, name string) error - // LoadKeyspaceID loads keyspace ID for the given keyspace specified by name. - LoadKeyspaceID(name string) (bool, uint32, error) + // SaveKeyspaceIDByName saves keyspace name to ID lookup information. + // It saves the ID onto the path encoded with name. + SaveKeyspaceIDByName(spaceID uint32, name string) error + // LoadKeyspaceIDByName loads keyspace ID for the given keyspace specified by name. + // It first constructs path to spaceID with the given name, then attempt to retrieve + // target spaceID. If the target keyspace does not exist, result boolean is set to false. + LoadKeyspaceIDByName(name string) (bool, uint32, error) } var _ KeyspaceStorage = (*StorageEndpoint)(nil) @@ -82,21 +96,22 @@ func (se *StorageEndpoint) LoadRangeKeyspace(startID uint32, limit int) ([]*keys return keyspaces, nil } -// SaveKeyspaceID saves keyspace name to ID lookup information to storage. -func (se *StorageEndpoint) SaveKeyspaceID(spaceID uint32, name string) error { +// SaveKeyspaceIDByName saves keyspace name to ID lookup information to storage. +func (se *StorageEndpoint) SaveKeyspaceIDByName(spaceID uint32, name string) error { key := KeyspaceIDPath(name) idStr := strconv.FormatUint(uint64(spaceID), spaceIDBase) return se.Save(key, idStr) } -// LoadKeyspaceID loads keyspace ID for the given keyspace name -func (se *StorageEndpoint) LoadKeyspaceID(name string) (bool, uint32, error) { +// LoadKeyspaceIDByName loads keyspace ID for the given keyspace name +func (se *StorageEndpoint) LoadKeyspaceIDByName(name string) (bool, uint32, error) { key := KeyspaceIDPath(name) idStr, err := se.Load(key) + // Failed to load the keyspaceID if loading operation errored, or if keyspace does not exist. if err != nil || idStr == "" { return false, 0, err } - id64, err := strconv.ParseUint(idStr, spaceIDBase, 32) + id64, err := strconv.ParseUint(idStr, spaceIDBase, spaceIDBitSizeMax) if err != nil { return false, 0, err } diff --git a/server/storage/keyspace_test.go b/server/storage/keyspace_test.go index b1cfeaae167..967a87c28e3 100644 --- a/server/storage/keyspace_test.go +++ b/server/storage/keyspace_test.go @@ -20,13 +20,14 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" + "github.com/tikv/pd/server/storage/endpoint" ) -func TestKeyspace(t *testing.T) { +func TestSaveLoadKeyspace(t *testing.T) { re := require.New(t) storage := NewStorageWithMemoryBackend() - keyspaces := testKeyspaces() + keyspaces := makeTestKeyspaces() for _, keyspace := range keyspaces { re.NoError(storage.SaveKeyspace(keyspace)) } @@ -53,22 +54,22 @@ func TestLoadRangeKeyspaces(t *testing.T) { re := require.New(t) storage := NewStorageWithMemoryBackend() - keyspaces := testKeyspaces() + keyspaces := makeTestKeyspaces() for _, keyspace := range keyspaces { re.NoError(storage.SaveKeyspace(keyspace)) } - // load all keyspaces. + // Load all keyspaces. loadedKeyspaces, err := storage.LoadRangeKeyspace(keyspaces[0].GetId(), 0) re.NoError(err) re.ElementsMatch(keyspaces, loadedKeyspaces) - // load keyspaces that with id no less than second test keyspace. + // Load keyspaces with id >= second test keyspace's id. loadedKeyspaces2, err := storage.LoadRangeKeyspace(keyspaces[1].GetId(), 0) re.NoError(err) re.ElementsMatch(keyspaces[1:], loadedKeyspaces2) - // load keyspace with the smallest id. + // Load keyspace with the smallest id. loadedKeyspace3, err := storage.LoadRangeKeyspace(1, 1) re.NoError(err) re.ElementsMatch(keyspaces[:1], loadedKeyspace3) @@ -81,27 +82,27 @@ func TestSaveLoadKeyspaceID(t *testing.T) { ids := []uint32{100, 200, 300} names := []string{"keyspace1", "keyspace2", "keyspace3"} for i := range ids { - re.NoError(storage.SaveKeyspaceID(ids[i], names[i])) + re.NoError(storage.SaveKeyspaceIDByName(ids[i], names[i])) } for i := range names { - success, id, err := storage.LoadKeyspaceID(names[i]) + success, id, err := storage.LoadKeyspaceIDByName(names[i]) re.NoError(err) re.True(success) re.Equal(ids[i], id) } - // loading non-existing id should return false, 0, nil - success, id, err := storage.LoadKeyspaceID("non-existing") + // Loading non-existing id should return false, 0, nil. + success, id, err := storage.LoadKeyspaceIDByName("non-existing") re.NoError(err) re.False(success) re.Equal(uint32(0), id) } -func testKeyspaces() []*keyspacepb.KeyspaceMeta { +func makeTestKeyspaces() []*keyspacepb.KeyspaceMeta { now := time.Now().Unix() return []*keyspacepb.KeyspaceMeta{ { - Id: 500, + Id: 10, Name: "keyspace1", State: keyspacepb.KeyspaceState_ENABLED, CreatedAt: now, @@ -112,7 +113,7 @@ func testKeyspaces() []*keyspacepb.KeyspaceMeta { }, }, { - Id: 700, + Id: 11, Name: "keyspace2", State: keyspacepb.KeyspaceState_ARCHIVED, CreatedAt: now + 300, @@ -123,7 +124,7 @@ func testKeyspaces() []*keyspacepb.KeyspaceMeta { }, }, { - Id: 800, + Id: 100, Name: "keyspace3", State: keyspacepb.KeyspaceState_DISABLED, CreatedAt: now + 500, @@ -135,3 +136,13 @@ func testKeyspaces() []*keyspacepb.KeyspaceMeta { }, } } + +// TestEncodeSpaceID test spaceID encoding. +func TestEncodeSpaceID(t *testing.T) { + re := require.New(t) + re.Equal("keyspaces/meta/00000000", endpoint.KeyspaceMetaPath(0)) + re.Equal("keyspaces/meta/16777215", endpoint.KeyspaceMetaPath(1<<24-1)) + re.Equal("keyspaces/meta/00000100", endpoint.KeyspaceMetaPath(100)) + re.Equal("keyspaces/meta/00000011", endpoint.KeyspaceMetaPath(11)) + re.Equal("keyspaces/meta/00000010", endpoint.KeyspaceMetaPath(10)) +} From 43632d93fb960026b4c730771d7c88c6724c5e05 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 20 Jul 2022 13:14:11 +0800 Subject: [PATCH 18/76] move helper functions to utils.go Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 46 +++-------------------- server/keyspace/keyspace_test.go | 24 ++++++++++++ server/keyspace/util.go | 64 ++++++++++++++++++++++++++++++++ server/keyspace/util_test.go | 64 ++++++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 41 deletions(-) create mode 100644 server/keyspace/util.go create mode 100644 server/keyspace/util_test.go diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index 3f6b29698ca..c53fa97a403 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -15,7 +15,6 @@ package keyspace import ( - "strings" "time" "github.com/pingcap/errors" @@ -26,28 +25,11 @@ import ( ) const ( - spaceIDMin = uint32(1) // 1 is the minimum value of spaceID, 0 is reserved. - spaceIDMax = ^uint32(0) >> 8 // 16777215 (Uint24Max) is the maximum value of spaceID. // AllocStep set idAllocator's step when write persistent window boundary. // Use a lower value for denser idAllocation in the event of frequent pd leader change. AllocStep = uint64(100) // AllocLabel is used to label keyspace idAllocator's metrics. AllocLabel = "keyspace-idAlloc" - // illegalChars contains forbidden characters for keyspace name. - illegalChars = "/" -) - -var ( - // ErrKeyspaceNotFound is used to indicate target keyspace does not exist. - ErrKeyspaceNotFound = errors.New("Keyspace does not exist") - // ErrKeyspaceExists indicates target keyspace already exists. - // Used when creating a new keyspace. - ErrKeyspaceExists = errors.New("Keyspace already exists") - errKeyspaceArchived = errors.New("Keyspace already archived") - errArchiveEnabled = errors.New("Cannot archive ENABLED keyspace") - errIllegalID = errors.New("Cannot create keyspace with that ID") - errIllegalName = errors.New("Cannot create keyspace with that name") - errIllegalOperation = errors.New("Illegal operation") ) // Manager manages keyspace related data. @@ -216,6 +198,11 @@ func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.Key // LoadRangeKeyspace load up to limit keyspaces starting from keyspace with startID. func (manager *Manager) LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { + // Load Start should fall within acceptable ID range, otherwise there may be problem with ID encoding. + // The only exception is 0, which is used to indicate beginning. + if err := validateID(startID); startID != 0 && err != nil { + return nil, err + } return manager.store.LoadRangeKeyspace(startID, limit) } @@ -236,29 +223,6 @@ func (manager *Manager) allocID() (uint32, error) { return id32, nil } -// validateID check if keyspace falls within the acceptable range. -// It throws errIllegalID when input id is our of range. -func validateID(spaceID uint32) error { - if spaceID < spaceIDMin || spaceID > spaceIDMax { - return errIllegalID - } - return nil -} - -// validateName check if name contains illegal character. -// It throws errIllegalName when name contains illegal character. -func validateName(name string) error { - // Name should not be empty. - if name == "" { - return errIllegalName - } - // Name should not contain any illegal character. - if strings.ContainsAny(name, illegalChars) { - return errIllegalName - } - return nil -} - // createNameToID create a keyspace name to ID lookup entry. // It returns error if saving keyspace name meet error or if name has already been taken. func (manager *Manager) createNameToID(spaceID uint32, name string) error { diff --git a/server/keyspace/keyspace_test.go b/server/keyspace/keyspace_test.go index ec5d2efbba7..739b32aacb9 100644 --- a/server/keyspace/keyspace_test.go +++ b/server/keyspace/keyspace_test.go @@ -16,6 +16,7 @@ package keyspace import ( "fmt" + "math" "strconv" "sync" "testing" @@ -160,6 +161,7 @@ func TestLoadRangeKeyspace(t *testing.T) { re := require.New(t) manager := newKeyspaceManager() // Test with 100 keyspaces. + // Keyspace ids are 1 - 101. total := 100 requests := makeCreateKeyspaceRequests(total) @@ -195,6 +197,28 @@ func TestLoadRangeKeyspace(t *testing.T) { re.Equal(uint32(loadStart+i), keyspaces[i].Id) checkCreateRequest(re, requests[i+loadStart-1], keyspaces[i]) } + + // Attempts to load 30 keyspaces starting from keyspace with id 90. + // Scan result should be keyspaces with id 90-101. + loadStart = 90 + keyspaces, err = manager.LoadRangeKeyspace(uint32(loadStart), 30) + re.NoError(err) + re.Equal(11, len(keyspaces)) + for i := range keyspaces { + re.Equal(uint32(loadStart+i), keyspaces[i].Id) + checkCreateRequest(re, requests[i+loadStart-1], keyspaces[i]) + } + + // Loading starting from non-existing keyspace ID should result in empty result. + loadStart = 900 + keyspaces, err = manager.LoadRangeKeyspace(uint32(loadStart), 0) + re.NoError(err) + re.Empty(keyspaces) + + // Scanning starting from a non-zero illegal index should result in error. + loadStart = math.MaxUint32 + _, err = manager.LoadRangeKeyspace(uint32(loadStart), 0) + re.Error(err) } // TestUpdateMultipleKeyspace checks that updating multiple keyspace's config simultaneously diff --git a/server/keyspace/util.go b/server/keyspace/util.go new file mode 100644 index 00000000000..ff95ea9b938 --- /dev/null +++ b/server/keyspace/util.go @@ -0,0 +1,64 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyspace + +import ( + "regexp" + + "github.com/pingcap/errors" +) + +const ( + spaceIDMin = uint32(1) // 1 is the minimum value of spaceID, 0 is reserved. + spaceIDMax = ^uint32(0) >> 8 // 16777215 (Uint24Max) is the maximum value of spaceID. + // namePattern is a regex that specifies acceptable characters of the keyspace name. + // Name must be non-empty and contains only alphanumerical, `_` and `-`. + namePattern = "^[-A-Za-z0-9_]+$" +) + +var ( + // ErrKeyspaceNotFound is used to indicate target keyspace does not exist. + ErrKeyspaceNotFound = errors.New("keyspace does not exist") + // ErrKeyspaceExists indicates target keyspace already exists. + // Used when creating a new keyspace. + ErrKeyspaceExists = errors.New("keyspace already exists") + errKeyspaceArchived = errors.New("keyspace already archived") + errArchiveEnabled = errors.New("cannot archive ENABLED keyspace") + errIllegalID = errors.New("illegal keyspace ID") + errIllegalName = errors.New("illegal keyspace name") + errIllegalOperation = errors.New("unknown operation") +) + +// validateID check if keyspace falls within the acceptable range. +// It throws errIllegalID when input id is our of range. +func validateID(spaceID uint32) error { + if spaceID < spaceIDMin || spaceID > spaceIDMax { + return errIllegalID + } + return nil +} + +// validateName check if name contains illegal character. +// It throws errIllegalName when name contains illegal character. +func validateName(name string) error { + isValid, err := regexp.MatchString(namePattern, name) + if err != nil { + return err + } + if !isValid { + return errIllegalName + } + return nil +} diff --git a/server/keyspace/util_test.go b/server/keyspace/util_test.go new file mode 100644 index 00000000000..620127b904f --- /dev/null +++ b/server/keyspace/util_test.go @@ -0,0 +1,64 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyspace + +import ( + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidateID(t *testing.T) { + re := require.New(t) + testCases := []struct { + id uint32 + hasErr bool + }{ + {0, true}, // Reserved keyspace should result in error. + {spaceIDMin - 1, true}, + {spaceIDMin, false}, + {spaceIDMin + 1, false}, + {100, false}, + {spaceIDMax - 1, false}, + {spaceIDMax, false}, + {spaceIDMax + 1, true}, + {math.MaxUint32, true}, + } + for _, testCase := range testCases { + re.Equal(testCase.hasErr, validateID(testCase.id) != nil) + } +} + +func TestValidateName(t *testing.T) { + re := require.New(t) + testCases := []struct { + name string + hasErr bool + }{ + {"keyspaceName1", false}, + {"keyspace_name_1", false}, + {"10", false}, + {"", true}, + {"keyspace/", true}, + {"keyspace:1", true}, + {"many many spaces", true}, + {"keyspace?limit=1", true}, + {"keyspace%1", true}, + } + for _, testCase := range testCases { + re.Equal(testCase.hasErr, validateName(testCase.name) != nil) + } +} From ff7718df843dac5b81ea46c7a29659efa4951dd9 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 20 Jul 2022 14:26:40 +0800 Subject: [PATCH 19/76] get keyspace manager Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace_service.go | 4 ++-- server/server.go | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 707148ac84c..9ea0d4229dd 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -75,7 +75,7 @@ func (s *KeyspaceServer) UpdateKeyspaceConfig(ctx context.Context, request *keys return &keyspacepb.UpdateKeyspaceConfigResponse{Header: s.notBootstrappedHeader()}, nil } - manager := s.keyspaceManager + manager := s.GetKeyspaceManager() updatedMeta, err := manager.UpdateKeyspaceConfig(request.Name, request.Mutations) if err != nil { return &keyspacepb.UpdateKeyspaceConfigResponse{Header: s.getErrorHeader(err)}, err @@ -93,7 +93,7 @@ func (s *KeyspaceServer) LoadKeyspace(ctx context.Context, request *keyspacepb.L return &keyspacepb.LoadKeyspaceResponse{Header: s.notBootstrappedHeader()}, nil } - manager := s.keyspaceManager + manager := s.GetKeyspaceManager() meta, err := manager.LoadKeyspace(request.Name) if err != nil { return &keyspacepb.LoadKeyspaceResponse{Header: s.getErrorHeader(err)}, err diff --git a/server/server.go b/server/server.go index b9e19a7033a..a63262dcfbe 100644 --- a/server/server.go +++ b/server/server.go @@ -803,6 +803,11 @@ func (s *Server) GetTSOAllocatorManager() *tso.AllocatorManager { return s.tsoAllocatorManager } +// GetKeyspaceManager returns the keyspace manager of server. +func (s *Server) GetKeyspaceManager() *keyspace.Manager { + return s.keyspaceManager +} + // Name returns the unique etcd Name for this server in etcd cluster. func (s *Server) Name() string { return s.cfg.Name From a97405dcac6650254284ef205b66d251ff99b8a4 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 20 Jul 2022 14:26:40 +0800 Subject: [PATCH 20/76] get keyspace manager Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace_service.go | 4 ++-- server/server.go | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 707148ac84c..9ea0d4229dd 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -75,7 +75,7 @@ func (s *KeyspaceServer) UpdateKeyspaceConfig(ctx context.Context, request *keys return &keyspacepb.UpdateKeyspaceConfigResponse{Header: s.notBootstrappedHeader()}, nil } - manager := s.keyspaceManager + manager := s.GetKeyspaceManager() updatedMeta, err := manager.UpdateKeyspaceConfig(request.Name, request.Mutations) if err != nil { return &keyspacepb.UpdateKeyspaceConfigResponse{Header: s.getErrorHeader(err)}, err @@ -93,7 +93,7 @@ func (s *KeyspaceServer) LoadKeyspace(ctx context.Context, request *keyspacepb.L return &keyspacepb.LoadKeyspaceResponse{Header: s.notBootstrappedHeader()}, nil } - manager := s.keyspaceManager + manager := s.GetKeyspaceManager() meta, err := manager.LoadKeyspace(request.Name) if err != nil { return &keyspacepb.LoadKeyspaceResponse{Header: s.getErrorHeader(err)}, err diff --git a/server/server.go b/server/server.go index b9e19a7033a..a63262dcfbe 100644 --- a/server/server.go +++ b/server/server.go @@ -803,6 +803,11 @@ func (s *Server) GetTSOAllocatorManager() *tso.AllocatorManager { return s.tsoAllocatorManager } +// GetKeyspaceManager returns the keyspace manager of server. +func (s *Server) GetKeyspaceManager() *keyspace.Manager { + return s.keyspaceManager +} + // Name returns the unique etcd Name for this server in etcd cluster. func (s *Server) Name() string { return s.cfg.Name From ee5a0288352d7dfbed891da7f35e83c8f2290f5e Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 20 Jul 2022 16:53:01 +0800 Subject: [PATCH 21/76] server: fix watch Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace_service.go | 13 +++-- tests/client/keyspace_test.go | 107 ++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 tests/client/keyspace_test.go diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 9ea0d4229dd..00d1bc4f321 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -16,11 +16,12 @@ package server import ( "context" - "github.com/gogo/protobuf/proto" - "github.com/tikv/pd/server/keyspace" + "path" + "github.com/gogo/protobuf/proto" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/tikv/pd/server/keyspace" "github.com/tikv/pd/server/storage/endpoint" "go.etcd.io/etcd/clientv3" ) @@ -117,7 +118,7 @@ func (s *KeyspaceServer) WatchKeyspaces(_ *keyspacepb.WatchKeyspacesRequest, str if err != nil { return err } - watchChan := s.client.Watch(ctx, endpoint.KeyspaceMetaPrefix(), clientv3.WithPrefix()) + watchChan := s.client.Watch(ctx, path.Join(s.rootPath, endpoint.KeyspaceMetaPrefix()), clientv3.WithPrefix()) for { select { case <-ctx.Done(): @@ -144,15 +145,17 @@ func (s *KeyspaceServer) WatchKeyspaces(_ *keyspacepb.WatchKeyspacesRequest, str } func (s *KeyspaceServer) sendAllKeyspaceMeta(ctx context.Context, stream keyspacepb.Keyspace_WatchKeyspacesServer) error { - getResp, err := s.client.Get(ctx, endpoint.KeyspaceMetaPrefix(), clientv3.WithPrefix()) + getResp, err := s.client.Get(ctx, path.Join(s.rootPath, endpoint.KeyspaceMetaPrefix()), clientv3.WithPrefix()) if err != nil { return err } metas := make([]*keyspacepb.KeyspaceMeta, getResp.Count) for i, kv := range getResp.Kvs { - if err = proto.Unmarshal(kv.Value, metas[i]); err != nil { + meta := &keyspacepb.KeyspaceMeta{} + if err = proto.Unmarshal(kv.Value, meta); err != nil { return err } + metas[i] = meta } return stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.header(), Keyspaces: metas}) } diff --git a/tests/client/keyspace_test.go b/tests/client/keyspace_test.go new file mode 100644 index 00000000000..2e716ce1ee5 --- /dev/null +++ b/tests/client/keyspace_test.go @@ -0,0 +1,107 @@ +// Copyright 2021 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client_test + +import ( + "fmt" + "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/stretchr/testify/require" + "github.com/tikv/pd/server" + "github.com/tikv/pd/server/keyspace" + "time" +) + +const ( + testConfig1 = "config_entry_1" + testConfig2 = "config_entry_2" +) + +func mustMakeTestKeyspaces(re *require.Assertions, server *server.Server, count int) []*keyspacepb.KeyspaceMeta { + now := time.Now() + var err error + keyspaces := make([]*keyspacepb.KeyspaceMeta, count) + manager := server.GetKeyspaceManager() + for i := 0; i < count; i++ { + keyspaces[i], err = manager.CreateKeyspace(&keyspace.CreateKeyspaceRequest{ + Name: fmt.Sprintf("test_keyspace%d", i), + InitialConfig: map[string]string{ + testConfig1: "100", + testConfig2: "200", + }, + Now: now, + }) + re.NoError(err) + } + return keyspaces +} +func (suite *clientTestSuite) TestLoadKeyspace() { + re := suite.Require() + metas := mustMakeTestKeyspaces(re, suite.srv, 10) + for _, expected := range metas { + loaded, err := suite.client.LoadKeyspace(suite.ctx, expected.Name) + re.NoError(err) + re.Equal(expected, loaded) + } + // Loading non-existing keyspace should result in error. + _, err := suite.client.LoadKeyspace(suite.ctx, "non-existing keyspace") + re.Error(err) +} + +func (suite *clientTestSuite) TestUpdateKeyspaceConfig() { + re := suite.Require() + metas := mustMakeTestKeyspaces(re, suite.srv, 10) + // Update keyspace configs. + for _, meta := range metas { + _, err := suite.client.UpdateKeyspaceConfig(suite.ctx, meta.Name, []*keyspacepb.Mutation{ + { + Op: keyspacepb.Op_PUT, + Key: []byte(testConfig1), + Value: []byte("new val"), + }, + { + Op: keyspacepb.Op_PUT, + Key: []byte("new config"), + Value: []byte("new val"), + }, + { + Op: keyspacepb.Op_DEL, + Key: []byte(testConfig2), + }, + }) + re.NoError(err) + } + // Verify updated keyspaces' configs matches the expectation. + expectedConfig := map[string]string{ + testConfig1: "new val", + "new config": "new val", + } + for _, meta := range metas { + loaded, err := suite.client.LoadKeyspace(suite.ctx, meta.Name) + re.NoError(err) + re.Equal(expectedConfig, loaded.Config) + } + // Updating a non-existing keyspace should result in error. + _, err := suite.client.UpdateKeyspaceConfig(suite.ctx, "non-existing keyspace", nil) + re.Error(err) +} + +func (suite *clientTestSuite) TestWatchKeyspace() { + re := suite.Require() + expected := mustMakeTestKeyspaces(re, suite.srv, 10) + watchChan, err := suite.client.WatchKeyspaces(suite.ctx) + re.NoError(err) + loaded := <-watchChan + re.Equal(expected, loaded) +} From f7e13fb4dc3fb2b0eef1da0fffba8a1b4b5e84c4 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 20 Jul 2022 16:53:27 +0800 Subject: [PATCH 22/76] client: added integration tests Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- client/go.mod | 2 +- client/go.sum | 4 ++-- client/keyspace_client.go | 12 +++++------ tests/client/go.mod | 2 ++ tests/client/go.sum | 13 ++---------- tests/client/keyspace_test.go | 40 +++++++++++++++++++++++++++++------ 6 files changed, 46 insertions(+), 27 deletions(-) diff --git a/client/go.mod b/client/go.mod index 5e51cbb3f9f..f86742524ac 100644 --- a/client/go.mod +++ b/client/go.mod @@ -3,7 +3,7 @@ module github.com/tikv/pd/client go 1.16 // TODO: Remove after kvproto merge -replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220707065914-b0848f38449c +replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6 require ( github.com/opentracing/opentracing-go v1.2.0 diff --git a/client/go.sum b/client/go.sum index 990e0f27e75..e5160ac2ea7 100644 --- a/client/go.sum +++ b/client/go.sum @@ -1,7 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/AmoebaProtozoa/kvproto v0.0.0-20220707065914-b0848f38449c h1:vdEMHupO5MP7D43Ym0klJwEA8VHAx3y1790klsH2e7s= -github.com/AmoebaProtozoa/kvproto v0.0.0-20220707065914-b0848f38449c/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6 h1:i9SJvU3P8t7MTEQ8zSUxDUDA3tyEf2+3I+JmWG5uw1Q= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/client/keyspace_client.go b/client/keyspace_client.go index b2128d6f3c4..f5debd56c27 100644 --- a/client/keyspace_client.go +++ b/client/keyspace_client.go @@ -30,7 +30,7 @@ import ( // KeyspaceClient manages keyspace metadata. type KeyspaceClient interface { // UpdateKeyspaceConfig updates target keyspace's config. - UpdateKeyspaceConfig(ctx context.Context, name string, put map[string]string, delete []string) (*keyspacepb.KeyspaceMeta, error) + UpdateKeyspaceConfig(ctx context.Context, name string, mutations []*keyspacepb.Mutation) (*keyspacepb.KeyspaceMeta, error) // LoadKeyspace load and return target keyspace's metadata. LoadKeyspace(ctx context.Context, name string) (*keyspacepb.KeyspaceMeta, error) // WatchKeyspaces watches keyspace meta changes. @@ -46,8 +46,7 @@ func (c *client) keyspaceClient() keyspacepb.KeyspaceClient { } // UpdateKeyspaceConfig updates target keyspace config and returns the updated keyspace meta. -// Note: delete will happen after put. -func (c *client) UpdateKeyspaceConfig(ctx context.Context, name string, put map[string]string, delete []string) (*keyspacepb.KeyspaceMeta, error) { +func (c *client) UpdateKeyspaceConfig(ctx context.Context, name string, mutations []*keyspacepb.Mutation) (*keyspacepb.KeyspaceMeta, error) { if span := opentracing.SpanFromContext(ctx); span != nil { span = opentracing.StartSpan("keyspaceClient.UpdateKeyspaceConfig", opentracing.ChildOf(span.Context())) defer span.Finish() @@ -56,10 +55,9 @@ func (c *client) UpdateKeyspaceConfig(ctx context.Context, name string, put map[ defer func() { cmdDurationUpdateKeyspaceConfig.Observe(time.Since(start).Seconds()) }() ctx, cancel := context.WithTimeout(ctx, c.option.timeout) req := &keyspacepb.UpdateKeyspaceConfigRequest{ - Header: c.requestHeader(), - Name: name, - Put: put, - Delete: delete, + Header: c.requestHeader(), + Name: name, + Mutations: mutations, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) resp, err := c.keyspaceClient().UpdateKeyspaceConfig(ctx, req) diff --git a/tests/client/go.mod b/tests/client/go.mod index 9d539193d52..e830af6987a 100644 --- a/tests/client/go.mod +++ b/tests/client/go.mod @@ -23,6 +23,8 @@ replace ( // reset grpc and protobuf deps in order to import client and server at the same time replace ( github.com/golang/protobuf v1.5.2 => github.com/golang/protobuf v1.3.4 + // TODO: Remove after kvproto merge + github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6 google.golang.org/grpc v1.43.0 => google.golang.org/grpc v1.26.0 google.golang.org/protobuf v1.26.0 => github.com/golang/protobuf v1.3.4 ) diff --git a/tests/client/go.sum b/tests/client/go.sum index ad49f3bc358..e608357ecb5 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -1,6 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AlekSi/gocov-xml v1.0.0/go.mod h1:J0qYeZ6tDg4oZubW9mAAgxlqw39PDfoEkzB3HXSbEuA= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6 h1:i9SJvU3P8t7MTEQ8zSUxDUDA3tyEf2+3I+JmWG5uw1Q= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -170,10 +172,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/goccy/go-graphviz v0.0.9 h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IPQQ= github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= @@ -187,7 +187,6 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -307,7 +306,6 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -415,10 +413,6 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= @@ -742,7 +736,6 @@ golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 h1:M73Iuj3xbbb9Uk1DYhzydths golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -783,11 +776,9 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= diff --git a/tests/client/keyspace_test.go b/tests/client/keyspace_test.go index 2e716ce1ee5..29355425827 100644 --- a/tests/client/keyspace_test.go +++ b/tests/client/keyspace_test.go @@ -28,14 +28,14 @@ const ( testConfig2 = "config_entry_2" ) -func mustMakeTestKeyspaces(re *require.Assertions, server *server.Server, count int) []*keyspacepb.KeyspaceMeta { +func mustMakeTestKeyspaces(re *require.Assertions, server *server.Server, start, count int) []*keyspacepb.KeyspaceMeta { now := time.Now() var err error keyspaces := make([]*keyspacepb.KeyspaceMeta, count) manager := server.GetKeyspaceManager() for i := 0; i < count; i++ { keyspaces[i], err = manager.CreateKeyspace(&keyspace.CreateKeyspaceRequest{ - Name: fmt.Sprintf("test_keyspace%d", i), + Name: fmt.Sprintf("test_keyspace%d", start+i), InitialConfig: map[string]string{ testConfig1: "100", testConfig2: "200", @@ -48,7 +48,7 @@ func mustMakeTestKeyspaces(re *require.Assertions, server *server.Server, count } func (suite *clientTestSuite) TestLoadKeyspace() { re := suite.Require() - metas := mustMakeTestKeyspaces(re, suite.srv, 10) + metas := mustMakeTestKeyspaces(re, suite.srv, 0, 10) for _, expected := range metas { loaded, err := suite.client.LoadKeyspace(suite.ctx, expected.Name) re.NoError(err) @@ -61,7 +61,7 @@ func (suite *clientTestSuite) TestLoadKeyspace() { func (suite *clientTestSuite) TestUpdateKeyspaceConfig() { re := suite.Require() - metas := mustMakeTestKeyspaces(re, suite.srv, 10) + metas := mustMakeTestKeyspaces(re, suite.srv, 0, 10) // Update keyspace configs. for _, meta := range metas { _, err := suite.client.UpdateKeyspaceConfig(suite.ctx, meta.Name, []*keyspacepb.Mutation{ @@ -99,9 +99,37 @@ func (suite *clientTestSuite) TestUpdateKeyspaceConfig() { func (suite *clientTestSuite) TestWatchKeyspace() { re := suite.Require() - expected := mustMakeTestKeyspaces(re, suite.srv, 10) + initialKeyspaces := mustMakeTestKeyspaces(re, suite.srv, 0, 10) watchChan, err := suite.client.WatchKeyspaces(suite.ctx) re.NoError(err) + // First batch of watchChan message should contain all existing keyspaces. + initialLoaded := <-watchChan + re.Equal(initialKeyspaces, initialLoaded) + // Each additional message contains extra put events. + additionalKeyspaces := mustMakeTestKeyspaces(re, suite.srv, 30, 10) + re.NoError(err) + // Checks that all additional keyspaces are captured by watch channel. + for i := 0; i < 10; { + loadedKeyspaces := <-watchChan + re.NotEmpty(loadedKeyspaces) + for j := range loadedKeyspaces { + re.Equal(additionalKeyspaces[i+j], loadedKeyspaces[j]) + } + i += len(loadedKeyspaces) + } + // Updates to state should also be captured. + expected, err := suite.srv.GetKeyspaceManager().UpdateKeyspaceState(initialKeyspaces[0].Name, keyspacepb.KeyspaceState_DISABLED, time.Now()) + re.NoError(err) loaded := <-watchChan - re.Equal(expected, loaded) + re.Equal([]*keyspacepb.KeyspaceMeta{expected}, loaded) + // Updates to config should also be captured. + expected, err = suite.srv.GetKeyspaceManager().UpdateKeyspaceConfig(initialKeyspaces[0].Name, []*keyspacepb.Mutation{ + { + Op: keyspacepb.Op_DEL, + Key: []byte(testConfig1), + }, + }) + re.NoError(err) + loaded = <-watchChan + re.Equal([]*keyspacepb.KeyspaceMeta{expected}, loaded) } From 90f8dbf8376f2e53df5db8ead946e61e32c446c6 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 20 Jul 2022 16:55:10 +0800 Subject: [PATCH 23/76] server: fix watch Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace_service.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 9ea0d4229dd..00d1bc4f321 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -16,11 +16,12 @@ package server import ( "context" - "github.com/gogo/protobuf/proto" - "github.com/tikv/pd/server/keyspace" + "path" + "github.com/gogo/protobuf/proto" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/tikv/pd/server/keyspace" "github.com/tikv/pd/server/storage/endpoint" "go.etcd.io/etcd/clientv3" ) @@ -117,7 +118,7 @@ func (s *KeyspaceServer) WatchKeyspaces(_ *keyspacepb.WatchKeyspacesRequest, str if err != nil { return err } - watchChan := s.client.Watch(ctx, endpoint.KeyspaceMetaPrefix(), clientv3.WithPrefix()) + watchChan := s.client.Watch(ctx, path.Join(s.rootPath, endpoint.KeyspaceMetaPrefix()), clientv3.WithPrefix()) for { select { case <-ctx.Done(): @@ -144,15 +145,17 @@ func (s *KeyspaceServer) WatchKeyspaces(_ *keyspacepb.WatchKeyspacesRequest, str } func (s *KeyspaceServer) sendAllKeyspaceMeta(ctx context.Context, stream keyspacepb.Keyspace_WatchKeyspacesServer) error { - getResp, err := s.client.Get(ctx, endpoint.KeyspaceMetaPrefix(), clientv3.WithPrefix()) + getResp, err := s.client.Get(ctx, path.Join(s.rootPath, endpoint.KeyspaceMetaPrefix()), clientv3.WithPrefix()) if err != nil { return err } metas := make([]*keyspacepb.KeyspaceMeta, getResp.Count) for i, kv := range getResp.Kvs { - if err = proto.Unmarshal(kv.Value, metas[i]); err != nil { + meta := &keyspacepb.KeyspaceMeta{} + if err = proto.Unmarshal(kv.Value, meta); err != nil { return err } + metas[i] = meta } return stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.header(), Keyspaces: metas}) } From 3af3d5511f24f50f7a46630164caa08e77c80881 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 21 Jul 2022 11:19:33 +0800 Subject: [PATCH 24/76] update update apis Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 129 +++++++++++++++++------------- 1 file changed, 74 insertions(+), 55 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index 4efaca8e1ac..1dd9bdf0da3 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -16,13 +16,11 @@ package handlers import ( "encoding/json" - "github.com/tikv/pd/server/apiv2/middlewares" "net/http" "strconv" "time" "github.com/gin-gonic/gin" - "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/server" @@ -31,11 +29,14 @@ import ( func RegisterKeyspace(r *gin.RouterGroup) { router := r.Group("keyspaces") - router.Use(middlewares.BootstrapChecker()) + //router.Use(middlewares.BootstrapChecker()) router.POST("", CreateKeyspace) router.GET("", LoadAllKeyspaces) router.GET("/:name", LoadKeyspace) - router.PATCH("/:name", UpdateKeyspace) + router.PATCH("/:name/updateConfig", UpdateKeyspaceConfig) + router.POST("/:name/enable", EnableKeyspace) + router.POST("/:name/disable", DisableKeyspace) + router.POST("/:name/archive", ArchiveKeyspace) } // CreateKeyspaceParams represents parameters needed when creating a new keyspace. @@ -187,42 +188,27 @@ func LoadAllKeyspaces(c *gin.Context) { c.IndentedJSON(http.StatusOK, resp) } -// UpdateKeyspaceParams represents parameters needed to update a keyspace. -// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. -type UpdateKeyspaceParams struct { - State string `json:"state"` - // Note: Config's values are string pointers. - // This is to differentiate between empty string "" and null value during binding. - // This is especially important when applying JSON merge patch, where null value means to remove, - // whereas empty string should simply set. - Config map[string]*string `json:"config"` -} - -// UpdateKeyspace update keyspace. +// UpdateKeyspaceConfig updates target keyspace's config. // @Tags keyspaces -// @Summary Update keyspace metadata. +// @Summary Update keyspace config. // @Param name path string true "Keyspace Name" -// @Param body body UpdateKeyspaceParams true "Update keyspace parameters" +// @Param body body map[string]*string true "Update keyspace parameters" // @Produce json // @Success 200 {object} KeyspaceMeta // @Failure 500 {string} string "PD server failed to proceed the request." -// Router /keyspaces/{name} [patch] -func UpdateKeyspace(c *gin.Context) { +// Router /keyspaces/{name}/updateConfig [patch] +func UpdateKeyspaceConfig(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() name := c.Param("name") - updateParams := &UpdateKeyspaceParams{} - err := c.BindJSON(updateParams) + mergePatch := map[string]*string{} + err := c.BindJSON(mergePatch) if err != nil { c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) return } - req, err := getUpdateRequest(name, updateParams) - if err != nil { - c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) - return - } - meta, err := manager.UpdateKeyspace(req) + mutations := getMutations(mergePatch) + meta, err := manager.UpdateKeyspaceConfig(name, mutations) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) return @@ -230,38 +216,71 @@ func UpdateKeyspace(c *gin.Context) { c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta}) } -// getUpdateRequest converts updateKeyspaceParams to keyspace.UpdateKeyspaceRequest. -func getUpdateRequest(name string, params *UpdateKeyspaceParams) (*keyspace.UpdateKeyspaceRequest, error) { - req := &keyspace.UpdateKeyspaceRequest{ - Name: name, - } - if params.State == "" { - req.UpdateState = false - } else { - req.UpdateState = true - req.Now = time.Now() - stateVal, ok := keyspacepb.KeyspaceState_value[params.State] - if !ok { - return nil, errors.New("Illegal keyspace state") - } - req.NewState = keyspacepb.KeyspaceState(stateVal) - } - toPut := map[string]string{} - var toDelete []string - for k, v := range params.Config { +// getMutations converts a given JSON merge patch to a series of keyspace config mutations. +func getMutations(patch map[string]*string) []*keyspacepb.Mutation { + mutations := make([]*keyspacepb.Mutation, 0, len(patch)) + for k, v := range patch { if v == nil { - toDelete = append(toDelete, k) + mutations = append(mutations, &keyspacepb.Mutation{ + Op: keyspacepb.Op_DEL, + Key: []byte(k), + }) } else { - toPut[k] = *v + mutations = append(mutations, &keyspacepb.Mutation{ + Op: keyspacepb.Op_PUT, + Key: []byte(k), + Value: []byte(*v), + }) } } - if len(toPut) > 0 { - req.ToPut = toPut - } - if len(toDelete) > 0 { - req.ToDelete = toDelete + return mutations +} + +// EnableKeyspace enables target keyspace. +// @Tags keyspaces +// @Summary Enable keyspace. +// @Param name path string true "Keyspace Name" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 500 {string} string "PD server failed to proceed the request." +// Router /keyspaces/{name}/enable [post] +func EnableKeyspace(c *gin.Context) { + UpdateKeyspaceState(c, keyspacepb.KeyspaceState_ENABLED) +} + +// DisableKeyspace disables target keyspace. +// @Tags keyspaces +// @Summary Disable keyspace. +// @Param name path string true "Keyspace Name" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 500 {string} string "PD server failed to proceed the request." +// Router /keyspaces/{name}/disable [post] +func DisableKeyspace(c *gin.Context) { + UpdateKeyspaceState(c, keyspacepb.KeyspaceState_DISABLED) +} + +// ArchiveKeyspace archives target keyspace. +// @Tags keyspaces +// @Summary Archive keyspace. +// @Param name path string true "Keyspace Name" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 500 {string} string "PD server failed to proceed the request." +// Router /keyspaces/{name}/archive [post] +func ArchiveKeyspace(c *gin.Context) { + UpdateKeyspaceState(c, keyspacepb.KeyspaceState_ARCHIVED) +} +func UpdateKeyspaceState(c *gin.Context, state keyspacepb.KeyspaceState) { + svr := c.MustGet("server").(*server.Server) + manager := svr.GetKeyspaceManager() + name := c.Param("name") + meta, err := manager.UpdateKeyspaceState(name, state, time.Now()) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return } - return req, nil + c.IndentedJSON(http.StatusOK, meta) } // KeyspaceMeta wraps keyspacepb.KeyspaceMeta to provide custom JSON marshal. From 09e852666d2145470fcbd2eeb60af61dd8e28e2e Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 21 Jul 2022 11:56:56 +0800 Subject: [PATCH 25/76] server: fix case where request does not contain initial config Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index c53fa97a403..2a2c9dd9d62 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -146,6 +146,9 @@ func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*keyspacep if keyspace.State == keyspacepb.KeyspaceState_ARCHIVED { return nil, errKeyspaceArchived } + if keyspace.Config == nil { + keyspace.Config = map[string]string{} + } // Update keyspace config according to mutations. for _, mutation := range mutations { switch mutation.Op { From 1ca5ebc720af37d981f12121ae878f7543373a44 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 21 Jul 2022 11:57:44 +0800 Subject: [PATCH 26/76] api: use updated design Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index 1dd9bdf0da3..4526507f143 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -188,11 +188,17 @@ func LoadAllKeyspaces(c *gin.Context) { c.IndentedJSON(http.StatusOK, resp) } +// UpdateConfigParams represents parameters needed to modify target keyspace's configs. +// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +type UpdateConfigParams struct { + Config map[string]*string `json:"config"` +} + // UpdateKeyspaceConfig updates target keyspace's config. // @Tags keyspaces // @Summary Update keyspace config. // @Param name path string true "Keyspace Name" -// @Param body body map[string]*string true "Update keyspace parameters" +// @Param body body UpdateConfigParams true "Update keyspace parameters" // @Produce json // @Success 200 {object} KeyspaceMeta // @Failure 500 {string} string "PD server failed to proceed the request." @@ -201,13 +207,13 @@ func UpdateKeyspaceConfig(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() name := c.Param("name") - mergePatch := map[string]*string{} - err := c.BindJSON(mergePatch) + configParams := &UpdateConfigParams{} + err := c.BindJSON(configParams) if err != nil { c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) return } - mutations := getMutations(mergePatch) + mutations := getMutations(configParams.Config) meta, err := manager.UpdateKeyspaceConfig(name, mutations) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) @@ -280,7 +286,7 @@ func UpdateKeyspaceState(c *gin.Context, state keyspacepb.KeyspaceState) { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) return } - c.IndentedJSON(http.StatusOK, meta) + c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta}) } // KeyspaceMeta wraps keyspacepb.KeyspaceMeta to provide custom JSON marshal. From 299f61272c806712907450c510dd56b202bc3aa0 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 21 Jul 2022 11:58:32 +0800 Subject: [PATCH 27/76] api: use updated design Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index 4526507f143..af74fbdca6d 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -24,12 +24,13 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/server" + "github.com/tikv/pd/server/apiv2/middlewares" "github.com/tikv/pd/server/keyspace" ) func RegisterKeyspace(r *gin.RouterGroup) { router := r.Group("keyspaces") - //router.Use(middlewares.BootstrapChecker()) + router.Use(middlewares.BootstrapChecker()) router.POST("", CreateKeyspace) router.GET("", LoadAllKeyspaces) router.GET("/:name", LoadKeyspace) From a983476ee040843178cfb4fb602e90b7c7d5f851 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 21 Jul 2022 16:24:22 +0800 Subject: [PATCH 28/76] added JSON unmarshall method to handler request Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index af74fbdca6d..988585ba14d 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -24,13 +24,11 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/apiv2/middlewares" "github.com/tikv/pd/server/keyspace" ) func RegisterKeyspace(r *gin.RouterGroup) { router := r.Group("keyspaces") - router.Use(middlewares.BootstrapChecker()) router.POST("", CreateKeyspace) router.GET("", LoadAllKeyspaces) router.GET("/:name", LoadKeyspace) @@ -313,3 +311,27 @@ func (meta *KeyspaceMeta) MarshalJSON() ([]byte, error) { meta.Config, }) } + +// UnmarshalJSON reverse KeyspaceMeta's the Custom JSON marshal. +func (meta *KeyspaceMeta) UnmarshalJSON(data []byte) error { + aux := &struct { + Name string `json:"name,omitempty"` + State string `json:"state,omitempty"` + CreatedAt int64 `json:"created_at,omitempty"` + StateChangedAt int64 `json:"state_changed_at,omitempty"` + Config map[string]string `json:"config,omitempty"` + }{} + + if err := json.Unmarshal(data, aux); err != nil { + return err + } + pbMeta := &keyspacepb.KeyspaceMeta{ + Name: aux.Name, + State: keyspacepb.KeyspaceState(keyspacepb.KeyspaceState_value[aux.State]), + CreatedAt: aux.CreatedAt, + StateChangedAt: aux.StateChangedAt, + Config: aux.Config, + } + meta.KeyspaceMeta = pbMeta + return nil +} From 2445ca1a5c223aca772c4745b1dd503dd610a6d5 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 21 Jul 2022 16:46:10 +0800 Subject: [PATCH 29/76] added API Integration tests Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 2 + tests/server/apiv2/handlers/keyspace_test.go | 180 +++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 tests/server/apiv2/handlers/keyspace_test.go diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index 988585ba14d..bb487a3d815 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -24,11 +24,13 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/server" + "github.com/tikv/pd/server/apiv2/middlewares" "github.com/tikv/pd/server/keyspace" ) func RegisterKeyspace(r *gin.RouterGroup) { router := r.Group("keyspaces") + router.Use(middlewares.BootstrapChecker()) router.POST("", CreateKeyspace) router.GET("", LoadAllKeyspaces) router.GET("/:name", LoadKeyspace) diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go new file mode 100644 index 00000000000..e4755dfe622 --- /dev/null +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -0,0 +1,180 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/testutil" + "github.com/tikv/pd/server/apiv2/handlers" + "github.com/tikv/pd/tests" + "go.uber.org/goleak" + "io" + "net/http" + "testing" +) + +const keyspacesPrefix = "/pd/api/v2/keyspaces" + +// dialClient used to dial http request. +var dialClient = &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + }, +} + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m, testutil.LeakOptions...) +} + +type keyspaceTestSuite struct { + suite.Suite + cleanup func() + cluster *tests.TestCluster + server *tests.TestServer +} + +func TestKeyspaceTestSuite(t *testing.T) { + suite.Run(t, new(keyspaceTestSuite)) +} + +func (suite *keyspaceTestSuite) SetupSuite() { + ctx, cancel := context.WithCancel(context.Background()) + suite.cleanup = cancel + cluster, err := tests.NewTestCluster(ctx, 3) + suite.cluster = cluster + suite.NoError(err) + suite.NoError(cluster.RunInitialServers()) + suite.NotEmpty(cluster.WaitLeader()) + suite.server = cluster.GetServer(cluster.GetLeader()) + suite.NoError(suite.server.BootstrapCluster()) +} + +func (suite *keyspaceTestSuite) TearDownSuite() { + suite.cleanup() + suite.cluster.Destroy() +} + +func (suite *keyspaceTestSuite) TestCreateLoadKeyspace() { + re := suite.Require() + keyspaces := mustMakeTestKeyspaces(re, suite.server, 0, 10) + for _, created := range keyspaces { + loaded := mustLoadKeyspaces(re, suite.server, created.Name) + re.Equal(created, loaded) + } +} + +func (suite *keyspaceTestSuite) TestUpdateKeyspaceConfig() { + re := suite.Require() + keyspaces := mustMakeTestKeyspaces(re, suite.server, 0, 10) + for _, created := range keyspaces { + config1val := "300" + updateRequest := &handlers.UpdateConfigParams{ + Config: map[string]*string{ + "config1": &config1val, + "config2": nil, + }, + } + updated := mustUpdateKeyspaceConfig(re, suite.server, created.Name, updateRequest) + checkUpdateRequest(re, updateRequest, created.Config, updated.Config) + } +} + +func mustMakeTestKeyspaces(re *require.Assertions, server *tests.TestServer, start, count int) []*keyspacepb.KeyspaceMeta { + testConfig := map[string]string{ + "config1": "100", + "config2": "200", + } + resultMeta := make([]*keyspacepb.KeyspaceMeta, count) + for i := 0; i < count; i++ { + createRequest := &handlers.CreateKeyspaceParams{ + Name: fmt.Sprintf("test_keyspace%d", start+i), + Config: testConfig, + } + resultMeta[i] = mustCreateKeyspace(re, server, createRequest) + } + return resultMeta +} + +func mustCreateKeyspace(re *require.Assertions, server *tests.TestServer, request *handlers.CreateKeyspaceParams) *keyspacepb.KeyspaceMeta { + data, err := json.Marshal(request) + re.NoError(err) + httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspacesPrefix, bytes.NewBuffer(data)) + resp, err := dialClient.Do(httpReq) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) + data, err = io.ReadAll(resp.Body) + re.NoError(err) + re.NoError(resp.Body.Close()) + meta := &handlers.KeyspaceMeta{} + re.NoError(json.Unmarshal(data, meta)) + checkCreateRequest(re, request, meta.KeyspaceMeta) + return meta.KeyspaceMeta +} + +func mustUpdateKeyspaceConfig(re *require.Assertions, server *tests.TestServer, name string, request *handlers.UpdateConfigParams) *keyspacepb.KeyspaceMeta { + data, err := json.Marshal(request) + re.NoError(err) + httpReq, err := http.NewRequest(http.MethodPatch, server.GetAddr()+keyspacesPrefix+"/"+name+"/updateConfig", bytes.NewBuffer(data)) + resp, err := dialClient.Do(httpReq) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) + data, err = io.ReadAll(resp.Body) + re.NoError(err) + re.NoError(resp.Body.Close()) + meta := &handlers.KeyspaceMeta{} + re.NoError(json.Unmarshal(data, meta)) + return meta.KeyspaceMeta +} + +func mustLoadKeyspaces(re *require.Assertions, server *tests.TestServer, name string) *keyspacepb.KeyspaceMeta { + resp, err := dialClient.Get(server.GetAddr() + keyspacesPrefix + "/" + name) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) + data, err := io.ReadAll(resp.Body) + re.NoError(err) + re.NoError(resp.Body.Close()) + meta := &handlers.KeyspaceMeta{} + re.NoError(json.Unmarshal(data, meta)) + return meta.KeyspaceMeta +} + +// checkCreateRequest verifies a keyspace meta matches a create request. +func checkCreateRequest(re *require.Assertions, request *handlers.CreateKeyspaceParams, meta *keyspacepb.KeyspaceMeta) { + re.Equal(request.Name, meta.Name) + re.Equal(keyspacepb.KeyspaceState_ENABLED, meta.State) + re.Equal(request.Config, meta.Config) +} + +// checkUpdateRequest verifies a keyspace meta matches a update request. +func checkUpdateRequest(re *require.Assertions, request *handlers.UpdateConfigParams, oldConfig, newConfig map[string]string) { + expected := map[string]string{} + for k, v := range oldConfig { + expected[k] = v + } + for k, v := range request.Config { + if v == nil { + delete(expected, k) + } else { + expected[k] = *v + } + } + re.Equal(expected, newConfig) +} From 32a29c6363ba5d639828522ddf952586d49e3d32 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 21 Jul 2022 18:10:23 +0800 Subject: [PATCH 30/76] update state integration test Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- tests/server/apiv2/handlers/keyspace_test.go | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index e4755dfe622..4667cbc8a37 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -97,6 +97,47 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceConfig() { } } +func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { + re := suite.Require() + keyspaces := mustMakeTestKeyspaces(re, suite.server, 0, 10) + for _, created := range keyspaces { + // Should not allow archiving enabled keyspace. + success, _ := sendUpdateStateRequest(re, suite.server, created.Name, "archive") + re.False(success) + // Disable an ENABLED keyspace is allowed. Should result in time stamp change. + success, disabled := sendUpdateStateRequest(re, suite.server, created.Name, "disable") + re.True(success) + re.Equal(keyspacepb.KeyspaceState_DISABLED, disabled.State) + re.NotEqual(created.StateChangedAt, disabled.StateChangedAt) + // Disable a already DISABLED keyspace should not result in any change. + success, disabledAgain := sendUpdateStateRequest(re, suite.server, created.Name, "disable") + re.True(success) + re.Equal(disabled, disabledAgain) + // Archiving a DISABLED keyspace should be allowed. Should result in time stamp change. + success, archived := sendUpdateStateRequest(re, suite.server, created.Name, "archive") + re.True(success) + re.Equal(keyspacepb.KeyspaceState_ARCHIVED, archived.State) + re.NotEqual(disabled.StateChangedAt, archived.StateChangedAt) + // Modifying ARCHIVED keyspace is not allowed. + success, _ = sendUpdateStateRequest(re, suite.server, created.Name, "disable") + re.False(success) + } +} + +func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, name, action string) (bool, *keyspacepb.KeyspaceMeta) { + httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspacesPrefix+"/"+name+"/"+action, nil) + re.NoError(err) + resp, err := dialClient.Do(httpReq) + if resp.StatusCode != http.StatusOK { + return false, nil + } + data, err := io.ReadAll(resp.Body) + re.NoError(err) + re.NoError(resp.Body.Close()) + meta := &handlers.KeyspaceMeta{} + re.NoError(json.Unmarshal(data, meta)) + return true, meta.KeyspaceMeta +} func mustMakeTestKeyspaces(re *require.Assertions, server *tests.TestServer, start, count int) []*keyspacepb.KeyspaceMeta { testConfig := map[string]string{ "config1": "100", From 7e0c73bef2b4b329387f0d4cd03679400c5e9b8f Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 22 Jul 2022 00:20:45 +0800 Subject: [PATCH 31/76] server: initi default keyspace Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- go.mod | 2 +- go.sum | 13 +---- server/keyspace/keyspace.go | 95 ++++++++++++++++++++++++-------- server/keyspace/keyspace_test.go | 91 +++++++++++++++++------------- server/keyspace/util.go | 18 ++++-- server/keyspace/util_test.go | 6 +- server/keyspace_service.go | 20 +------ server/server.go | 5 +- 8 files changed, 148 insertions(+), 102 deletions(-) diff --git a/go.mod b/go.mod index 124055b0fcc..9cc763d265d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/tikv/pd go 1.16 // TODO: Remove after kvproto merge -replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6 +replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220721101402-e48eba4cc8f7 require ( github.com/AlekSi/gocov-xml v1.0.0 diff --git a/go.sum b/go.sum index b4595910a9d..903c45771e9 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AlekSi/gocov-xml v1.0.0 h1:4QctJBgXEkbzeKz6PJy6bt3JSPNSN4I2mITYW+eKUoQ= github.com/AlekSi/gocov-xml v1.0.0/go.mod h1:J0qYeZ6tDg4oZubW9mAAgxlqw39PDfoEkzB3HXSbEuA= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220721101402-e48eba4cc8f7 h1:l8MbmqS/q+3slfDRsFa3w0yTJbS+uSxgkZL9QOPzkJA= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220721101402-e48eba4cc8f7/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -178,10 +180,8 @@ github.com/goccy/go-graphviz v0.0.9 h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IP github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= @@ -195,7 +195,6 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -305,7 +304,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -414,10 +412,6 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMtVcOkjUcuQKh+YrluSo7+7YMCQSzy30= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= -github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= @@ -729,7 +723,6 @@ golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 h1:M73Iuj3xbbb9Uk1DYhzydths golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -771,11 +764,9 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index c53fa97a403..ab1ec2e18a9 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -29,7 +29,9 @@ const ( // Use a lower value for denser idAllocation in the event of frequent pd leader change. AllocStep = uint64(100) // AllocLabel is used to label keyspace idAllocator's metrics. - AllocLabel = "keyspace-idAlloc" + AllocLabel = "keyspace-idAlloc" + defaultKeyspaceName = "DEFAULT" + defaultKeyspaceID = uint32(0) ) // Manager manages keyspace related data. @@ -49,20 +51,48 @@ type Manager struct { type CreateKeyspaceRequest struct { // Name of the keyspace to be created. // Using an existing name will result in error. - Name string - InitialConfig map[string]string - Now time.Time + Name string + Config map[string]string + Now time.Time } // NewKeyspaceManager creates a Manager of keyspace related data. -func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator) *Manager { - return &Manager{ +func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator) (*Manager, error) { + manager := &Manager{ store: store, idAllocator: idAllocator, } + + manager.metaLock.Lock() + defer manager.metaLock.Unlock() + // Check if default keyspace already exists. + defaultExists, err := manager.store.LoadKeyspace(defaultKeyspaceID, &keyspacepb.KeyspaceMeta{}) + if err != nil { + return nil, err + } + if defaultExists { + return manager, nil + } + // Initialize default keyspace. + defaultKeyspace := &keyspacepb.KeyspaceMeta{ + Id: defaultKeyspaceID, + Name: defaultKeyspaceName, + State: keyspacepb.KeyspaceState_ENABLED, + } + if err = manager.store.SaveKeyspace(defaultKeyspace); err != nil { + return nil, err + } + if err = manager.createNameToID(defaultKeyspaceID, defaultKeyspaceName); err != nil { + if removeErr := manager.store.RemoveKeyspace(defaultKeyspaceID); removeErr != nil { + return nil, errors.Wrap(removeErr, "failed to remove keyspace meta after save spaceID failure") + } + return nil, err + } + + return manager, nil } -// CreateKeyspace create a keyspace meta with initial config and save it to storage. +// CreateKeyspace create a keyspace meta with given config and save it to storage. func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { // Validate purposed name's legality. if err := validateName(request.Name); err != nil { @@ -81,20 +111,20 @@ func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspac State: keyspacepb.KeyspaceState_ENABLED, CreatedAt: request.Now.Unix(), StateChangedAt: request.Now.Unix(), - Config: request.InitialConfig, + Config: request.Config, } manager.metaLock.Lock() defer manager.metaLock.Unlock() // Check if keyspace with that id already exists. - idExists, err := manager.store.LoadKeyspace(newID, &keyspacepb.KeyspaceMeta{}) + keyspaceExists, err := manager.store.LoadKeyspace(newID, &keyspacepb.KeyspaceMeta{}) if err != nil { return nil, err } - if idExists { + if keyspaceExists { return nil, ErrKeyspaceExists } // Save keyspace meta before saving id. - if err := manager.store.SaveKeyspace(keyspace); err != nil { + if err = manager.store.SaveKeyspace(keyspace); err != nil { return nil, err } // Create name to ID entry, @@ -132,9 +162,22 @@ func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, err return keyspace, nil } -// UpdateKeyspaceConfig apply mutations to target keyspace in order. +type Mutation struct { + Op OpType + Key string + Value string +} + +type OpType int + +const ( + OpPut OpType = iota + 1 // Operation type starts at 1. + OpDel +) + +// UpdateKeyspaceConfig changes target keyspace's config in the order specified in mutations. // It returns error if saving failed, operation not allowed, or if keyspace not exists. -func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*keyspacepb.Mutation) (*keyspacepb.KeyspaceMeta, error) { +func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation) (*keyspacepb.KeyspaceMeta, error) { manager.metaLock.Lock() defer manager.metaLock.Unlock() // Load keyspace by name. @@ -146,13 +189,16 @@ func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*keyspacep if keyspace.State == keyspacepb.KeyspaceState_ARCHIVED { return nil, errKeyspaceArchived } + if keyspace.Config == nil { + keyspace.Config = map[string]string{} + } // Update keyspace config according to mutations. for _, mutation := range mutations { switch mutation.Op { - case keyspacepb.Op_PUT: - keyspace.Config[string(mutation.Key)] = string(mutation.Value) - case keyspacepb.Op_DEL: - delete(keyspace.Config, string(mutation.Key)) + case OpPut: + keyspace.Config[mutation.Key] = mutation.Value + case OpDel: + delete(keyspace.Config, mutation.Key) default: return nil, errIllegalOperation } @@ -167,6 +213,10 @@ func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*keyspacep // UpdateKeyspaceState updates target keyspace to the given state if it's not already in that state. // It returns error if saving failed, operation not allowed, or if keyspace not exists. func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.KeyspaceState, now time.Time) (*keyspacepb.KeyspaceMeta, error) { + // Changing the state of default keyspace is not allowed. + if name == defaultKeyspaceName { + return nil, errModifyDefault + } manager.metaLock.Lock() defer manager.metaLock.Unlock() // Load keyspace by name. @@ -198,10 +248,9 @@ func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.Key // LoadRangeKeyspace load up to limit keyspaces starting from keyspace with startID. func (manager *Manager) LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { - // Load Start should fall within acceptable ID range, otherwise there may be problem with ID encoding. - // The only exception is 0, which is used to indicate beginning. - if err := validateID(startID); startID != 0 && err != nil { - return nil, err + // Load Start should fall within acceptable ID range. + if startID > spaceIDMax { + return nil, errIllegalID } return manager.store.LoadRangeKeyspace(startID, limit) } @@ -213,8 +262,8 @@ func (manager *Manager) allocID() (uint32, error) { return 0, err } id32 := uint32(id64) - // If obtained id is too small, re-allocate a higher ID. - if id32 < spaceIDMin { + // Skip reserved space ID. + if id32 == defaultKeyspaceID { return manager.allocID() } if err = validateID(id32); err != nil { diff --git a/server/keyspace/keyspace_test.go b/server/keyspace/keyspace_test.go index 739b32aacb9..11ed42e2257 100644 --- a/server/keyspace/keyspace_test.go +++ b/server/keyspace/keyspace_test.go @@ -35,10 +35,12 @@ const ( testConfig2 = "config_entry_2" ) -func newKeyspaceManager() *Manager { +func mustNewKeyspaceManager(re *require.Assertions) *Manager { store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) allocator := mockid.NewIDAllocator() - return NewKeyspaceManager(store, allocator) + manager, err := NewKeyspaceManager(store, allocator) + re.NoError(err) + return manager } func makeCreateKeyspaceRequests(count int) []*CreateKeyspaceRequest { @@ -47,7 +49,7 @@ func makeCreateKeyspaceRequests(count int) []*CreateKeyspaceRequest { for i := 0; i < count; i++ { requests[i] = &CreateKeyspaceRequest{ Name: fmt.Sprintf("test_keyspace%d", i), - InitialConfig: map[string]string{ + Config: map[string]string{ testConfig1: "100", testConfig2: "200", }, @@ -59,7 +61,7 @@ func makeCreateKeyspaceRequests(count int) []*CreateKeyspaceRequest { func TestCreateKeyspace(t *testing.T) { re := require.New(t) - manager := newKeyspaceManager() + manager := mustNewKeyspaceManager(re) requests := makeCreateKeyspaceRequests(10) for i, request := range requests { @@ -83,36 +85,36 @@ func TestCreateKeyspace(t *testing.T) { re.Error(err) } -func makeMutations() []*keyspacepb.Mutation { - return []*keyspacepb.Mutation{ +func makeMutations() []*Mutation { + return []*Mutation{ { - Op: keyspacepb.Op_PUT, - Key: []byte(testConfig1), - Value: []byte("new val"), + Op: OpPut, + Key: testConfig1, + Value: "new val", }, { - Op: keyspacepb.Op_PUT, - Key: []byte("new config"), - Value: []byte("new val"), + Op: OpPut, + Key: "new config", + Value: "new val", }, { - Op: keyspacepb.Op_DEL, - Key: []byte(testConfig2), + Op: OpDel, + Key: testConfig2, }, } } func TestUpdateKeyspaceConfig(t *testing.T) { re := require.New(t) - manager := newKeyspaceManager() + manager := mustNewKeyspaceManager(re) requests := makeCreateKeyspaceRequests(5) + mutations := makeMutations() for _, createRequest := range requests { _, err := manager.CreateKeyspace(createRequest) re.NoError(err) - mutations := makeMutations() updated, err := manager.UpdateKeyspaceConfig(createRequest.Name, mutations) re.NoError(err) - checkMutations(re, createRequest.InitialConfig, updated.Config, mutations) + checkMutations(re, createRequest.Config, updated.Config, mutations) // Changing config of a ARCHIVED keyspace is not allowed. _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_DISABLED, time.Now()) re.NoError(err) @@ -121,11 +123,15 @@ func TestUpdateKeyspaceConfig(t *testing.T) { _, err = manager.UpdateKeyspaceConfig(createRequest.Name, mutations) re.Error(err) } + // Changing config of DEFAULT keyspace is allowed. + updated, err := manager.UpdateKeyspaceConfig(defaultKeyspaceName, mutations) + re.NoError(err) + checkMutations(re, nil, updated.Config, mutations) } func TestUpdateKeyspaceState(t *testing.T) { re := require.New(t) - manager := newKeyspaceManager() + manager := mustNewKeyspaceManager(re) requests := makeCreateKeyspaceRequests(5) for _, createRequest := range requests { _, err := manager.CreateKeyspace(createRequest) @@ -154,14 +160,17 @@ func TestUpdateKeyspaceState(t *testing.T) { // Changing state of an ARCHIVED keyspace is not allowed. _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ENABLED, newTime) re.Error(err) + // Changing state of DEFAULT keyspace is not allowed. + _, err = manager.UpdateKeyspaceState(defaultKeyspaceName, keyspacepb.KeyspaceState_DISABLED, time.Now()) + re.Error(err) } } func TestLoadRangeKeyspace(t *testing.T) { re := require.New(t) - manager := newKeyspaceManager() + manager := mustNewKeyspaceManager(re) // Test with 100 keyspaces. - // Keyspace ids are 1 - 101. + // Created keyspace ids are 1 - 100. total := 100 requests := makeCreateKeyspaceRequests(total) @@ -170,25 +179,31 @@ func TestLoadRangeKeyspace(t *testing.T) { re.NoError(err) } - // Load all keyspaces. + // Load all keyspaces including the default keyspace. keyspaces, err := manager.LoadRangeKeyspace(0, 0) re.NoError(err) - re.Equal(total, len(keyspaces)) + re.Equal(total+1, len(keyspaces)) for i := range keyspaces { - re.Equal(uint32(i+1), keyspaces[i].Id) - checkCreateRequest(re, requests[i], keyspaces[i]) + re.Equal(uint32(i), keyspaces[i].Id) + if i != 0 { + checkCreateRequest(re, requests[i-1], keyspaces[i]) + } } // Load first 50 keyspaces. + // Result should be keyspaces with id 0 - 49. keyspaces, err = manager.LoadRangeKeyspace(0, 50) re.NoError(err) re.Equal(50, len(keyspaces)) for i := range keyspaces { - re.Equal(uint32(i+1), keyspaces[i].Id) - checkCreateRequest(re, requests[i], keyspaces[i]) + re.Equal(uint32(i), keyspaces[i].Id) + if i != 0 { + checkCreateRequest(re, requests[i-1], keyspaces[i]) + } } // Load 20 keyspaces starting from keyspace with id 33. + // Result should be keyspaces with id 33 - 52. loadStart := 33 keyspaces, err = manager.LoadRangeKeyspace(uint32(loadStart), 20) re.NoError(err) @@ -199,7 +214,7 @@ func TestLoadRangeKeyspace(t *testing.T) { } // Attempts to load 30 keyspaces starting from keyspace with id 90. - // Scan result should be keyspaces with id 90-101. + // Scan result should be keyspaces with id 90-100. loadStart = 90 keyspaces, err = manager.LoadRangeKeyspace(uint32(loadStart), 30) re.NoError(err) @@ -225,7 +240,7 @@ func TestLoadRangeKeyspace(t *testing.T) { // will be successful. func TestUpdateMultipleKeyspace(t *testing.T) { re := require.New(t) - manager := newKeyspaceManager() + manager := mustNewKeyspaceManager(re) requests := makeCreateKeyspaceRequests(50) for _, createRequest := range requests { _, err := manager.CreateKeyspace(createRequest) @@ -258,11 +273,11 @@ func checkCreateRequest(re *require.Assertions, request *CreateKeyspaceRequest, re.Equal(request.Now.Unix(), meta.CreatedAt) re.Equal(request.Now.Unix(), meta.StateChangedAt) re.Equal(keyspacepb.KeyspaceState_ENABLED, meta.State) - re.Equal(request.InitialConfig, meta.Config) + re.Equal(request.Config, meta.Config) } // checkMutations verifies that performing mutations on old config would result in new config. -func checkMutations(re *require.Assertions, oldConfig, newConfig map[string]string, mutations []*keyspacepb.Mutation) { +func checkMutations(re *require.Assertions, oldConfig, newConfig map[string]string, mutations []*Mutation) { // Copy oldConfig to expected to avoid modifying its content. expected := map[string]string{} for k, v := range oldConfig { @@ -270,10 +285,10 @@ func checkMutations(re *require.Assertions, oldConfig, newConfig map[string]stri } for _, mutation := range mutations { switch mutation.Op { - case keyspacepb.Op_PUT: - expected[string(mutation.Key)] = string(mutation.Value) - case keyspacepb.Op_DEL: - delete(expected, string(mutation.Key)) + case OpPut: + expected[mutation.Key] = mutation.Value + case OpDel: + delete(expected, mutation.Key) } } re.Equal(expected, newConfig) @@ -284,11 +299,11 @@ func updateKeyspaceConfig(re *require.Assertions, manager *Manager, name string, oldMeta, err := manager.LoadKeyspace(name) re.NoError(err) for i := 0; i <= end; i++ { - mutations := []*keyspacepb.Mutation{ + mutations := []*Mutation{ { - Op: keyspacepb.Op_PUT, - Key: []byte(testConfig), - Value: []byte(strconv.Itoa(i)), + Op: OpPut, + Key: testConfig, + Value: strconv.Itoa(i), }, } updatedMeta, err := manager.UpdateKeyspaceConfig(name, mutations) diff --git a/server/keyspace/util.go b/server/keyspace/util.go index ff95ea9b938..6cd57b08ca9 100644 --- a/server/keyspace/util.go +++ b/server/keyspace/util.go @@ -21,7 +21,6 @@ import ( ) const ( - spaceIDMin = uint32(1) // 1 is the minimum value of spaceID, 0 is reserved. spaceIDMax = ^uint32(0) >> 8 // 16777215 (Uint24Max) is the maximum value of spaceID. // namePattern is a regex that specifies acceptable characters of the keyspace name. // Name must be non-empty and contains only alphanumerical, `_` and `-`. @@ -36,22 +35,28 @@ var ( ErrKeyspaceExists = errors.New("keyspace already exists") errKeyspaceArchived = errors.New("keyspace already archived") errArchiveEnabled = errors.New("cannot archive ENABLED keyspace") + errModifyDefault = errors.New("cannot modify default keyspace's state") errIllegalID = errors.New("illegal keyspace ID") errIllegalName = errors.New("illegal keyspace name") errIllegalOperation = errors.New("unknown operation") ) // validateID check if keyspace falls within the acceptable range. -// It throws errIllegalID when input id is our of range. +// It throws errIllegalID when input id is our of range, +// or if it collides with reserved id. func validateID(spaceID uint32) error { - if spaceID < spaceIDMin || spaceID > spaceIDMax { + if spaceID > spaceIDMax { + return errIllegalID + } + if spaceID == defaultKeyspaceID { return errIllegalID } return nil } -// validateName check if name contains illegal character. -// It throws errIllegalName when name contains illegal character. +// validateName check if user provided name is legal. +// It throws errIllegalName when name contains illegal character, +// or if it collides with reserved name. func validateName(name string) error { isValid, err := regexp.MatchString(namePattern, name) if err != nil { @@ -60,5 +65,8 @@ func validateName(name string) error { if !isValid { return errIllegalName } + if name == defaultKeyspaceName { + return errIllegalName + } return nil } diff --git a/server/keyspace/util_test.go b/server/keyspace/util_test.go index 620127b904f..e4e59924e00 100644 --- a/server/keyspace/util_test.go +++ b/server/keyspace/util_test.go @@ -27,10 +27,7 @@ func TestValidateID(t *testing.T) { id uint32 hasErr bool }{ - {0, true}, // Reserved keyspace should result in error. - {spaceIDMin - 1, true}, - {spaceIDMin, false}, - {spaceIDMin + 1, false}, + {0, true}, // Reserved id should result in error. {100, false}, {spaceIDMax - 1, false}, {spaceIDMax, false}, @@ -48,6 +45,7 @@ func TestValidateName(t *testing.T) { name string hasErr bool }{ + {"DEFAULT", true}, // Reserved name should result in error. {"keyspaceName1", false}, {"keyspace_name_1", false}, {"10", false}, diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 00d1bc4f321..0db09fd45b5 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -70,25 +70,7 @@ func (s *KeyspaceServer) getErrorHeader(err error) *pdpb.ResponseHeader { } } -func (s *KeyspaceServer) UpdateKeyspaceConfig(ctx context.Context, request *keyspacepb.UpdateKeyspaceConfigRequest) (*keyspacepb.UpdateKeyspaceConfigResponse, error) { - rc := s.GetRaftCluster() - if rc == nil { - return &keyspacepb.UpdateKeyspaceConfigResponse{Header: s.notBootstrappedHeader()}, nil - } - - manager := s.GetKeyspaceManager() - updatedMeta, err := manager.UpdateKeyspaceConfig(request.Name, request.Mutations) - if err != nil { - return &keyspacepb.UpdateKeyspaceConfigResponse{Header: s.getErrorHeader(err)}, err - } - - return &keyspacepb.UpdateKeyspaceConfigResponse{ - Header: s.header(), - Keyspace: updatedMeta, - }, nil -} - -func (s *KeyspaceServer) LoadKeyspace(ctx context.Context, request *keyspacepb.LoadKeyspaceRequest) (*keyspacepb.LoadKeyspaceResponse, error) { +func (s *KeyspaceServer) LoadKeyspace(_ context.Context, request *keyspacepb.LoadKeyspaceRequest) (*keyspacepb.LoadKeyspaceResponse, error) { rc := s.GetRaftCluster() if rc == nil { return &keyspacepb.LoadKeyspaceResponse{Header: s.notBootstrappedHeader()}, nil diff --git a/server/server.go b/server/server.go index 18ed2689c30..49c80c05e33 100644 --- a/server/server.go +++ b/server/server.go @@ -425,7 +425,10 @@ func (s *Server) startServer(ctx context.Context) error { Member: s.member.MemberValue(), Step: keyspace.AllocStep, }) - s.keyspaceManager = keyspace.NewKeyspaceManager(s.storage, keyspaceIDAllocator) + s.keyspaceManager, err = keyspace.NewKeyspaceManager(s.storage, keyspaceIDAllocator) + if err != nil { + return err + } s.basicCluster = core.NewBasicCluster() s.cluster = cluster.NewRaftCluster(ctx, s.clusterID, syncer.NewRegionSyncer(s), s.client, s.httpClient) s.hbStreams = hbstream.NewHeartbeatStreams(ctx, s.clusterID, s.cluster) From b3fce3569277d7f47075cda8bc3014537169b587 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 22 Jul 2022 10:32:21 +0800 Subject: [PATCH 32/76] client: removed update config rpc, added default keyspace tests Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- client/go.mod | 2 +- client/go.sum | 4 +- client/keyspace_client.go | 34 ----------------- client/metrics.go | 2 - tests/client/go.mod | 2 +- tests/client/go.sum | 13 +------ tests/client/keyspace_test.go | 69 +++++++++++++---------------------- 7 files changed, 31 insertions(+), 95 deletions(-) diff --git a/client/go.mod b/client/go.mod index f86742524ac..73ad5397e92 100644 --- a/client/go.mod +++ b/client/go.mod @@ -3,7 +3,7 @@ module github.com/tikv/pd/client go 1.16 // TODO: Remove after kvproto merge -replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6 +replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220721101402-e48eba4cc8f7 require ( github.com/opentracing/opentracing-go v1.2.0 diff --git a/client/go.sum b/client/go.sum index e5160ac2ea7..18c34a9097a 100644 --- a/client/go.sum +++ b/client/go.sum @@ -1,7 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6 h1:i9SJvU3P8t7MTEQ8zSUxDUDA3tyEf2+3I+JmWG5uw1Q= -github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220721101402-e48eba4cc8f7 h1:l8MbmqS/q+3slfDRsFa3w0yTJbS+uSxgkZL9QOPzkJA= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220721101402-e48eba4cc8f7/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/client/keyspace_client.go b/client/keyspace_client.go index f5debd56c27..a469a45b00e 100644 --- a/client/keyspace_client.go +++ b/client/keyspace_client.go @@ -29,8 +29,6 @@ import ( // KeyspaceClient manages keyspace metadata. type KeyspaceClient interface { - // UpdateKeyspaceConfig updates target keyspace's config. - UpdateKeyspaceConfig(ctx context.Context, name string, mutations []*keyspacepb.Mutation) (*keyspacepb.KeyspaceMeta, error) // LoadKeyspace load and return target keyspace's metadata. LoadKeyspace(ctx context.Context, name string) (*keyspacepb.KeyspaceMeta, error) // WatchKeyspaces watches keyspace meta changes. @@ -45,38 +43,6 @@ func (c *client) keyspaceClient() keyspacepb.KeyspaceClient { return nil } -// UpdateKeyspaceConfig updates target keyspace config and returns the updated keyspace meta. -func (c *client) UpdateKeyspaceConfig(ctx context.Context, name string, mutations []*keyspacepb.Mutation) (*keyspacepb.KeyspaceMeta, error) { - if span := opentracing.SpanFromContext(ctx); span != nil { - span = opentracing.StartSpan("keyspaceClient.UpdateKeyspaceConfig", opentracing.ChildOf(span.Context())) - defer span.Finish() - } - start := time.Now() - defer func() { cmdDurationUpdateKeyspaceConfig.Observe(time.Since(start).Seconds()) }() - ctx, cancel := context.WithTimeout(ctx, c.option.timeout) - req := &keyspacepb.UpdateKeyspaceConfigRequest{ - Header: c.requestHeader(), - Name: name, - Mutations: mutations, - } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.keyspaceClient().UpdateKeyspaceConfig(ctx, req) - cancel() - - if err != nil { - cmdFailedDurationUpdateKeyspaceConfig.Observe(time.Since(start).Seconds()) - c.ScheduleCheckLeader() - return nil, err - } - - if resp.Header.GetError() != nil { - cmdFailedDurationUpdateKeyspaceConfig.Observe(time.Since(start).Seconds()) - return nil, errors.Errorf("update keyspace %s config failed: %s", name, resp.Header.GetError().String()) - } - - return resp.Keyspace, nil -} - // LoadKeyspace loads and returns target keyspace's metadata. func (c *client) LoadKeyspace(ctx context.Context, name string) (*keyspacepb.KeyspaceMeta, error) { if span := opentracing.SpanFromContext(ctx); span != nil { diff --git a/client/metrics.go b/client/metrics.go index 2e3eae8bc3f..b92b66727d6 100644 --- a/client/metrics.go +++ b/client/metrics.go @@ -99,7 +99,6 @@ var ( cmdDurationGetOperator = cmdDuration.WithLabelValues("get_operator") cmdDurationSplitRegions = cmdDuration.WithLabelValues("split_regions") cmdDurationSplitAndScatterRegions = cmdDuration.WithLabelValues("split_and_scatter_regions") - cmdDurationUpdateKeyspaceConfig = cmdDuration.WithLabelValues("update_keyspace_config") cmdDurationLoadKeyspace = cmdDuration.WithLabelValues("load_keyspace") cmdFailDurationGetRegion = cmdFailedDuration.WithLabelValues("get_region") @@ -112,7 +111,6 @@ var ( cmdFailedDurationGetAllStores = cmdFailedDuration.WithLabelValues("get_all_stores") cmdFailedDurationUpdateGCSafePoint = cmdFailedDuration.WithLabelValues("update_gc_safe_point") cmdFailedDurationUpdateServiceGCSafePoint = cmdFailedDuration.WithLabelValues("update_service_gc_safe_point") - cmdFailedDurationUpdateKeyspaceConfig = cmdDuration.WithLabelValues("update_keyspace_config") cmdFailedDurationLoadKeyspace = cmdDuration.WithLabelValues("load_keyspace") requestDurationTSO = requestDuration.WithLabelValues("tso") ) diff --git a/tests/client/go.mod b/tests/client/go.mod index e830af6987a..f3f12f4012b 100644 --- a/tests/client/go.mod +++ b/tests/client/go.mod @@ -24,7 +24,7 @@ replace ( replace ( github.com/golang/protobuf v1.5.2 => github.com/golang/protobuf v1.3.4 // TODO: Remove after kvproto merge - github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220715053401-9a717ffca7c6 + github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20220721101402-e48eba4cc8f7 google.golang.org/grpc v1.43.0 => google.golang.org/grpc v1.26.0 google.golang.org/protobuf v1.26.0 => github.com/golang/protobuf v1.3.4 ) diff --git a/tests/client/go.sum b/tests/client/go.sum index e0c9200a704..592218ffbcc 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -1,6 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AlekSi/gocov-xml v1.0.0/go.mod h1:J0qYeZ6tDg4oZubW9mAAgxlqw39PDfoEkzB3HXSbEuA= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220721101402-e48eba4cc8f7 h1:l8MbmqS/q+3slfDRsFa3w0yTJbS+uSxgkZL9QOPzkJA= +github.com/AmoebaProtozoa/kvproto v0.0.0-20220721101402-e48eba4cc8f7/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -175,10 +177,8 @@ github.com/goccy/go-graphviz v0.0.9 h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IP github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= @@ -192,7 +192,6 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -314,7 +313,6 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -422,10 +420,6 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= @@ -752,7 +746,6 @@ golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 h1:M73Iuj3xbbb9Uk1DYhzydths golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -794,11 +787,9 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= diff --git a/tests/client/keyspace_test.go b/tests/client/keyspace_test.go index 29355425827..2e8d5ec1e6d 100644 --- a/tests/client/keyspace_test.go +++ b/tests/client/keyspace_test.go @@ -36,7 +36,7 @@ func mustMakeTestKeyspaces(re *require.Assertions, server *server.Server, start, for i := 0; i < count; i++ { keyspaces[i], err = manager.CreateKeyspace(&keyspace.CreateKeyspaceRequest{ Name: fmt.Sprintf("test_keyspace%d", start+i), - InitialConfig: map[string]string{ + Config: map[string]string{ testConfig1: "100", testConfig2: "200", }, @@ -57,44 +57,11 @@ func (suite *clientTestSuite) TestLoadKeyspace() { // Loading non-existing keyspace should result in error. _, err := suite.client.LoadKeyspace(suite.ctx, "non-existing keyspace") re.Error(err) -} - -func (suite *clientTestSuite) TestUpdateKeyspaceConfig() { - re := suite.Require() - metas := mustMakeTestKeyspaces(re, suite.srv, 0, 10) - // Update keyspace configs. - for _, meta := range metas { - _, err := suite.client.UpdateKeyspaceConfig(suite.ctx, meta.Name, []*keyspacepb.Mutation{ - { - Op: keyspacepb.Op_PUT, - Key: []byte(testConfig1), - Value: []byte("new val"), - }, - { - Op: keyspacepb.Op_PUT, - Key: []byte("new config"), - Value: []byte("new val"), - }, - { - Op: keyspacepb.Op_DEL, - Key: []byte(testConfig2), - }, - }) - re.NoError(err) - } - // Verify updated keyspaces' configs matches the expectation. - expectedConfig := map[string]string{ - testConfig1: "new val", - "new config": "new val", - } - for _, meta := range metas { - loaded, err := suite.client.LoadKeyspace(suite.ctx, meta.Name) - re.NoError(err) - re.Equal(expectedConfig, loaded.Config) - } - // Updating a non-existing keyspace should result in error. - _, err := suite.client.UpdateKeyspaceConfig(suite.ctx, "non-existing keyspace", nil) - re.Error(err) + // Loading default keyspace should be successful. + keyspaceDefault, err := suite.client.LoadKeyspace(suite.ctx, "DEFAULT") + re.NoError(err) + re.Equal(uint32(0), keyspaceDefault.Id) + re.Equal("DEFAULT", keyspaceDefault.Name) } func (suite *clientTestSuite) TestWatchKeyspace() { @@ -102,9 +69,12 @@ func (suite *clientTestSuite) TestWatchKeyspace() { initialKeyspaces := mustMakeTestKeyspaces(re, suite.srv, 0, 10) watchChan, err := suite.client.WatchKeyspaces(suite.ctx) re.NoError(err) - // First batch of watchChan message should contain all existing keyspaces. + // First batch of watchChan message should contain all existing keyspaces, including the default. initialLoaded := <-watchChan - re.Equal(initialKeyspaces, initialLoaded) + re.Equal(len(initialKeyspaces)+1, len(initialLoaded)) + for i := range initialKeyspaces { + re.Equal(initialKeyspaces[i], initialLoaded[i+1]) + } // Each additional message contains extra put events. additionalKeyspaces := mustMakeTestKeyspaces(re, suite.srv, 30, 10) re.NoError(err) @@ -123,10 +93,21 @@ func (suite *clientTestSuite) TestWatchKeyspace() { loaded := <-watchChan re.Equal([]*keyspacepb.KeyspaceMeta{expected}, loaded) // Updates to config should also be captured. - expected, err = suite.srv.GetKeyspaceManager().UpdateKeyspaceConfig(initialKeyspaces[0].Name, []*keyspacepb.Mutation{ + expected, err = suite.srv.GetKeyspaceManager().UpdateKeyspaceConfig(initialKeyspaces[0].Name, []*keyspace.Mutation{ + { + Op: keyspace.OpDel, + Key: testConfig1, + }, + }) + re.NoError(err) + loaded = <-watchChan + re.Equal([]*keyspacepb.KeyspaceMeta{expected}, loaded) + // Updates to default keyspace's config should also be captured. + expected, err = suite.srv.GetKeyspaceManager().UpdateKeyspaceConfig("DEFAULT", []*keyspace.Mutation{ { - Op: keyspacepb.Op_DEL, - Key: []byte(testConfig1), + Op: keyspace.OpPut, + Key: "config", + Value: "value", }, }) re.NoError(err) From fbe2581280f194ee20f85767cf601d77cf8a40ba Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 22 Jul 2022 11:22:09 +0800 Subject: [PATCH 33/76] api: fix tests Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 26 +++++++++++--------- tests/server/apiv2/handlers/keyspace_test.go | 18 ++++++++------ 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index bb487a3d815..260ab495c0f 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -65,9 +65,9 @@ func CreateKeyspace(c *gin.Context) { return } req := &keyspace.CreateKeyspaceRequest{ - Name: createParams.Name, - InitialConfig: createParams.Config, - Now: time.Now(), + Name: createParams.Name, + Config: createParams.Config, + Now: time.Now(), } meta, err := manager.CreateKeyspace(req) if err != nil { @@ -191,6 +191,8 @@ func LoadAllKeyspaces(c *gin.Context) { // UpdateConfigParams represents parameters needed to modify target keyspace's configs. // NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +// A Map of string to string pointer is used to differentiate between json null and "", +// which will both be set to "" if value type is string during binding. type UpdateConfigParams struct { Config map[string]*string `json:"config"` } @@ -224,19 +226,19 @@ func UpdateKeyspaceConfig(c *gin.Context) { } // getMutations converts a given JSON merge patch to a series of keyspace config mutations. -func getMutations(patch map[string]*string) []*keyspacepb.Mutation { - mutations := make([]*keyspacepb.Mutation, 0, len(patch)) +func getMutations(patch map[string]*string) []*keyspace.Mutation { + mutations := make([]*keyspace.Mutation, 0, len(patch)) for k, v := range patch { if v == nil { - mutations = append(mutations, &keyspacepb.Mutation{ - Op: keyspacepb.Op_DEL, - Key: []byte(k), + mutations = append(mutations, &keyspace.Mutation{ + Op: keyspace.OpDel, + Key: k, }) } else { - mutations = append(mutations, &keyspacepb.Mutation{ - Op: keyspacepb.Op_PUT, - Key: []byte(k), - Value: []byte(*v), + mutations = append(mutations, &keyspace.Mutation{ + Op: keyspace.OpPut, + Key: k, + Value: *v, }) } } diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index 4667cbc8a37..dc61ddeaee4 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -83,7 +83,7 @@ func (suite *keyspaceTestSuite) TestCreateLoadKeyspace() { func (suite *keyspaceTestSuite) TestUpdateKeyspaceConfig() { re := suite.Require() - keyspaces := mustMakeTestKeyspaces(re, suite.server, 0, 10) + keyspaces := mustMakeTestKeyspaces(re, suite.server, 10, 10) for _, created := range keyspaces { config1val := "300" updateRequest := &handlers.UpdateConfigParams{ @@ -99,29 +99,30 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceConfig() { func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { re := suite.Require() - keyspaces := mustMakeTestKeyspaces(re, suite.server, 0, 10) + keyspaces := mustMakeTestKeyspaces(re, suite.server, 20, 10) for _, created := range keyspaces { - // Should not allow archiving enabled keyspace. + // Should NOT allow archiving ENABLED keyspace. success, _ := sendUpdateStateRequest(re, suite.server, created.Name, "archive") re.False(success) - // Disable an ENABLED keyspace is allowed. Should result in time stamp change. + // Disabling an ENABLED keyspace is allowed. success, disabled := sendUpdateStateRequest(re, suite.server, created.Name, "disable") re.True(success) re.Equal(keyspacepb.KeyspaceState_DISABLED, disabled.State) - re.NotEqual(created.StateChangedAt, disabled.StateChangedAt) - // Disable a already DISABLED keyspace should not result in any change. + // Disabling an already DISABLED keyspace should not result in any change. success, disabledAgain := sendUpdateStateRequest(re, suite.server, created.Name, "disable") re.True(success) re.Equal(disabled, disabledAgain) - // Archiving a DISABLED keyspace should be allowed. Should result in time stamp change. + // Archiving a DISABLED keyspace should be allowed. success, archived := sendUpdateStateRequest(re, suite.server, created.Name, "archive") re.True(success) re.Equal(keyspacepb.KeyspaceState_ARCHIVED, archived.State) - re.NotEqual(disabled.StateChangedAt, archived.StateChangedAt) // Modifying ARCHIVED keyspace is not allowed. success, _ = sendUpdateStateRequest(re, suite.server, created.Name, "disable") re.False(success) } + // Changing default keyspace's state is NOT allowed. + success, _ := sendUpdateStateRequest(re, suite.server, "DEFAULT", "disable") + re.False(success) } func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, name, action string) (bool, *keyspacepb.KeyspaceMeta) { @@ -149,6 +150,7 @@ func mustMakeTestKeyspaces(re *require.Assertions, server *tests.TestServer, sta Name: fmt.Sprintf("test_keyspace%d", start+i), Config: testConfig, } + fmt.Println(createRequest) resultMeta[i] = mustCreateKeyspace(re, server, createRequest) } return resultMeta From dec67b560a6d351f423f6539c62c1283e0aa5331 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 22 Jul 2022 14:13:57 +0800 Subject: [PATCH 34/76] added RunInTxn style interface to base kv Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/kv/etcd_kv.go | 95 ++++++++++++++++++++++++++++++ server/storage/kv/kv.go | 29 ++++++++-- server/storage/kv/levedb_kv.go | 69 ++++++++++++++++++++++ server/storage/kv/mem_kv.go | 102 +++++++++++++++++++++++++++++++++ 4 files changed, 291 insertions(+), 4 deletions(-) diff --git a/server/storage/kv/etcd_kv.go b/server/storage/kv/etcd_kv.go index e30b5f7b462..aee156988d9 100644 --- a/server/storage/kv/etcd_kv.go +++ b/server/storage/kv/etcd_kv.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/etcdutil" + "github.com/tikv/pd/pkg/syncutil" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" ) @@ -171,3 +172,97 @@ func (t *SlowLogTxn) Commit() (*clientv3.TxnResponse, error) { return resp, errors.WithStack(err) } + +// etcdTxn is used to record user's action during RunInTxn, +// It stores load result in conditions and modification in operations. +type etcdTxn struct { + kv *etcdKVBase + ctx context.Context + // mu protects conditions and operations. + mu syncutil.Mutex + conditions []clientv3.Cmp + operations []clientv3.Op +} + +// RunInTxn runs user provided function f in a transaction. +func (kv *etcdKVBase) RunInTxn(ctx context.Context, f func(txn Txn) error) error { + txn := &etcdTxn{ + kv: kv, + ctx: ctx, + } + err := f(txn) + if err != nil { + return err + } + return txn.commit() +} + +// Save puts a put operation into operations. +func (txn *etcdTxn) Save(key, value string) error { + key = path.Join(txn.kv.rootPath, key) + operation := clientv3.OpPut(key, value) + txn.mu.Lock() + defer txn.mu.Unlock() + txn.operations = append(txn.operations, operation) + return nil +} + +// Remove puts a delete operation into operations. +func (txn *etcdTxn) Remove(key string) error { + key = path.Join(txn.kv.rootPath, key) + operation := clientv3.OpDelete(key) + txn.mu.Lock() + defer txn.mu.Unlock() + txn.operations = append(txn.operations, operation) + return nil +} + +// Load loads the target value from etcd and puts a comparator into conditions. +func (txn *etcdTxn) Load(key string) (string, error) { + value, err := txn.kv.Load(key) + // If Load failed, preserve the failure behavior of base Load. + if err != nil { + return value, err + } + // If load successful, must make sure value stays the same before commit. + fullKey := path.Join(txn.kv.rootPath, key) + condition := clientv3.Compare(clientv3.Value(fullKey), "=", value) + txn.mu.Lock() + defer txn.mu.Unlock() + txn.conditions = append(txn.conditions, condition) + return value, err +} + +// LoadRange loads the target range from etcd, +// Then for each value loaded, it puts a comparator into conditions. +func (txn *etcdTxn) LoadRange(key, endKey string, limit int) (keys []string, values []string, err error) { + keys, values, err = txn.kv.LoadRange(key, endKey, limit) + // If LoadRange failed, preserve the failure behavior of base LoadRange. + if err != nil { + return keys, values, err + } + // If LoadRange successful, must make sure values stay the same before commit. + txn.mu.Lock() + defer txn.mu.Unlock() + for i := range keys { + fullKey := path.Join(txn.kv.rootPath, keys[i]) + condition := clientv3.Compare(clientv3.Value(fullKey), "=", values[i]) + txn.conditions = append(txn.conditions, condition) + } + return keys, values, err +} + +// commit perform the operations on etcd, with pre-condition that values observed by user has not been changed. +func (txn *etcdTxn) commit() error { + baseTxn := txn.kv.client.Txn(txn.ctx) + baseTxn.If(txn.conditions...) + baseTxn.Then(txn.operations...) + resp, err := baseTxn.Commit() + if err != nil { + return err + } + if !resp.Succeeded { + return errs.ErrEtcdTxnConflict.FastGenByArgs() + } + return nil +} diff --git a/server/storage/kv/kv.go b/server/storage/kv/kv.go index 2f1fa06e144..f5338aebd25 100644 --- a/server/storage/kv/kv.go +++ b/server/storage/kv/kv.go @@ -14,10 +14,31 @@ package kv -// Base is an abstract interface for load/save pd cluster data. -type Base interface { - Load(key string) (string, error) - LoadRange(key, endKey string, limit int) (keys []string, values []string, err error) +import "context" + +// Txn bundles multiple operations into a single executable unit. +// It enables kv to atomically apply a set of updates. +type Txn interface { Save(key, value string) error Remove(key string) error + Load(key string) (string, error) + LoadRange(key, endKey string, limit int) (keys []string, values []string, err error) +} + +// Base is an abstract interface for load/save pd cluster data. +type Base interface { + Txn + // RunInTxn runs the user provided function in a Transaction. + // If user provided function f returns a non-nil error, then + // transaction will not be committed, the same error will be + // returned by RunInTxn. + // Otherwise, it returns the error occurred during the + // transaction. + // Note: + // 1. Load and LoadRange operations provides only stale read. + // Values saved/ removed during transaction will not be immediately + // observable in the same transaction. + // 2. Only when storage is etcd, does RunInTxn checks that + // values loaded during transaction has not been modified before commit. + RunInTxn(ctx context.Context, f func(txn Txn) error) error } diff --git a/server/storage/kv/levedb_kv.go b/server/storage/kv/levedb_kv.go index 7f134709bd1..64207aea2d0 100644 --- a/server/storage/kv/levedb_kv.go +++ b/server/storage/kv/levedb_kv.go @@ -15,12 +15,15 @@ package kv import ( + "context" + "github.com/gogo/protobuf/proto" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/syncutil" ) // LevelDBKV is a kv store using LevelDB. @@ -94,3 +97,69 @@ func (kv *LevelDBKV) SaveRegions(regions map[string]*metapb.Region) error { } return nil } + +type levelDBTxn struct { + kv *LevelDBKV + ctx context.Context + // mu protects batch. + mu syncutil.Mutex + batch *leveldb.Batch +} + +// RunInTxn runs user provided function f in a transaction. +// If user provided function returns error, then transaction will not be committed. +func (kv *LevelDBKV) RunInTxn(ctx context.Context, f func(txn Txn) error) error { + txn := &levelDBTxn{ + kv: kv, + ctx: ctx, + batch: new(leveldb.Batch), + } + err := f(txn) + if err != nil { + return err + } + return txn.commit() +} + +// Save puts a save operation with target key value into levelDB batch. +func (txn *levelDBTxn) Save(key, value string) error { + txn.mu.Lock() + defer txn.mu.Unlock() + + txn.batch.Put([]byte(key), []byte(value)) + return nil +} + +// Remove puts a delete operation with target key into levelDB batch. +func (txn *levelDBTxn) Remove(key string) error { + txn.mu.Lock() + defer txn.mu.Unlock() + + txn.batch.Delete([]byte(key)) + return nil +} + +// Load executes base's load. +func (txn *levelDBTxn) Load(key string) (string, error) { + return txn.kv.Load(key) +} + +// LoadRange executes base's load range. +func (txn *levelDBTxn) LoadRange(key, endKey string, limit int) (keys []string, values []string, err error) { + return txn.kv.LoadRange(key, endKey, limit) +} + +// commit writes the batch constructed into levelDB. +func (txn *levelDBTxn) commit() error { + // Check context first to make sure transaction is not cancelled. + select { + default: + case <-txn.ctx.Done(): + return txn.ctx.Err() + } + + txn.mu.Lock() + defer txn.mu.Unlock() + + return txn.kv.Write(txn.batch, nil) +} diff --git a/server/storage/kv/mem_kv.go b/server/storage/kv/mem_kv.go index b74cab84b11..9f98b23b48f 100644 --- a/server/storage/kv/mem_kv.go +++ b/server/storage/kv/mem_kv.go @@ -15,6 +15,8 @@ package kv import ( + "context" + "github.com/google/btree" "github.com/pingcap/errors" "github.com/pingcap/failpoint" @@ -87,3 +89,103 @@ func (kv *memoryKV) Remove(key string) error { kv.tree.Delete(memoryKVItem{key, ""}) return nil } + +// memTxn implements kv.Txn +type memTxn struct { + kv *memoryKV + ctx context.Context + // mu protects ops. + mu syncutil.Mutex + ops []*op +} + +// op represents an Operation that memKV can execute. +type op struct { + t opType + key string + val string +} + +type opType int + +const ( + tPut opType = iota + tDelete +) + +// RunInTxn runs the user provided function f in a transaction. +// If user provided function returns error, then transaction will not be committed. +func (kv *memoryKV) RunInTxn(ctx context.Context, f func(txn Txn) error) error { + txn := &memTxn{ + kv: kv, + ctx: ctx, + } + err := f(txn) + if err != nil { + return err + } + return txn.commit() +} + +// Save appends a save operation to ops. +func (txn *memTxn) Save(key, value string) error { + txn.mu.Lock() + defer txn.mu.Unlock() + + txn.ops = append(txn.ops, &op{ + t: tPut, + key: key, + val: value, + }) + return nil +} + +// Remove appends a remove operation to ops. +func (txn *memTxn) Remove(key string) error { + txn.mu.Lock() + defer txn.mu.Unlock() + + txn.ops = append(txn.ops, &op{ + t: tDelete, + key: key, + }) + return nil +} + +// Load executes base's load directly. +func (txn *memTxn) Load(key string) (string, error) { + return txn.kv.Load(key) +} + +// LoadRange executes base's load range directly. +func (txn *memTxn) LoadRange(key, endKey string, limit int) (keys []string, values []string, err error) { + return txn.kv.LoadRange(key, endKey, limit) +} + +// commit executes operations in ops. +func (txn *memTxn) commit() error { + // Check context first to make sure transaction is not cancelled. + select { + default: + case <-txn.ctx.Done(): + return txn.ctx.Err() + } + // Lock txn.mu to protect memTxn ops. + txn.mu.Lock() + defer txn.mu.Unlock() + // Lock kv.lock to protect the execution of the batch, + // making the execution atomic. + txn.kv.Lock() + defer txn.kv.Unlock() + // Execute mutations in order. + // Note: executions in mem_kv never fails. + for _, op := range txn.ops { + switch op.t { + case tPut: + txn.kv.tree.ReplaceOrInsert(memoryKVItem{op.key, op.val}) + case tDelete: + txn.kv.tree.Delete(memoryKVItem{op.key, ""}) + } + } + return nil +} From 898b2d1cda26528ed49d0470768cb2a2326605d4 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 22 Jul 2022 14:54:38 +0800 Subject: [PATCH 35/76] added unit tests Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/kv/kv_test.go | 69 ++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/server/storage/kv/kv_test.go b/server/storage/kv/kv_test.go index bf9f3604264..a453bf3126a 100644 --- a/server/storage/kv/kv_test.go +++ b/server/storage/kv/kv_test.go @@ -15,6 +15,7 @@ package kv import ( + "context" "fmt" "net/url" "path" @@ -45,8 +46,29 @@ func TestEtcd(t *testing.T) { kv := NewEtcdKVBase(client, rootPath) testReadWrite(re, kv) testRange(re, kv) + testSaveMultiple(re, kv, 20) + testLoadConflict(re, kv) } +func TestEtcdRunInTxn(t *testing.T) { + re := require.New(t) + cfg := newTestSingleConfig(t) + etcd, err := embed.StartEtcd(cfg) + re.NoError(err) + defer etcd.Close() + + ep := cfg.LCUrls[0].String() + client, err := clientv3.New(clientv3.Config{ + Endpoints: []string{ep}, + }) + re.NoError(err) + rootPath := path.Join("/pd", strconv.FormatUint(100, 10)) + + kv := NewEtcdKVBase(client, rootPath) + testReadWrite(re, kv) + testRange(re, kv) + testSaveMultiple(re, kv, 20) +} func TestLevelDB(t *testing.T) { re := require.New(t) dir := t.TempDir() @@ -62,6 +84,7 @@ func TestMemKV(t *testing.T) { kv := NewMemoryKV() testReadWrite(re, kv) testRange(re, kv) + testSaveMultiple(re, kv, 20) } func testReadWrite(re *require.Assertions, kv Base) { @@ -137,3 +160,49 @@ func newTestSingleConfig(t *testing.T) *embed.Config { cfg.ClusterState = embed.ClusterStateFlagNew return cfg } + +func testSaveMultiple(re *require.Assertions, kv Base, count int) { + var err error + err = kv.RunInTxn(context.Background(), func(txn Txn) error { + var saveErr error + for i := 0; i < count; i++ { + saveErr = txn.Save("key"+strconv.Itoa(i), "val"+strconv.Itoa(i)) + if saveErr != nil { + return saveErr + } + } + return nil + }) + re.NoError(err) + for i := 0; i < count; i++ { + val, loadErr := kv.Load("key" + strconv.Itoa(i)) + re.NoError(loadErr) + re.Equal("val"+strconv.Itoa(i), val) + } +} + +func testLoadConflict(re *require.Assertions, kv Base) { + re.NoError(kv.Save("testKey", "initialValue")) + // loader loads the test key value. + loader := func(txn Txn) error { + _, err := txn.Load("testKey") + if err != nil { + return err + } + return nil + } + // When no other writer, loader must succeed. + re.NoError(kv.RunInTxn(context.Background(), loader)) + + conflictLoader := func(txn Txn) error { + _, err := txn.Load("testKey") + // update key after load. + re.NoError(kv.Save("testKey", "newValue")) + if err != nil { + return err + } + return nil + } + // When other writer exists, loader must error. + re.Error(kv.RunInTxn(context.Background(), conflictLoader)) +} From 290487b67b8d1311fb747fe270dd7428ca43ed36 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 22 Jul 2022 16:43:58 +0800 Subject: [PATCH 36/76] server: export default keyspace name and id Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 22 ++++++++++++---------- server/keyspace/keyspace_test.go | 4 ++-- server/keyspace/util.go | 4 ++-- server/keyspace/util_test.go | 4 ++-- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index ab1ec2e18a9..ec5874f3578 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -29,9 +29,11 @@ const ( // Use a lower value for denser idAllocation in the event of frequent pd leader change. AllocStep = uint64(100) // AllocLabel is used to label keyspace idAllocator's metrics. - AllocLabel = "keyspace-idAlloc" - defaultKeyspaceName = "DEFAULT" - defaultKeyspaceID = uint32(0) + AllocLabel = "keyspace-idAlloc" + // DefaultKeyspaceName is the name reserved for default keyspace. + DefaultKeyspaceName = "DEFAULT" + // DefaultKeyspaceID is the id of default keyspace. + DefaultKeyspaceID = uint32(0) ) // Manager manages keyspace related data. @@ -66,7 +68,7 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator manager.metaLock.Lock() defer manager.metaLock.Unlock() // Check if default keyspace already exists. - defaultExists, err := manager.store.LoadKeyspace(defaultKeyspaceID, &keyspacepb.KeyspaceMeta{}) + defaultExists, err := manager.store.LoadKeyspace(DefaultKeyspaceID, &keyspacepb.KeyspaceMeta{}) if err != nil { return nil, err } @@ -75,15 +77,15 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator } // Initialize default keyspace. defaultKeyspace := &keyspacepb.KeyspaceMeta{ - Id: defaultKeyspaceID, - Name: defaultKeyspaceName, + Id: DefaultKeyspaceID, + Name: DefaultKeyspaceName, State: keyspacepb.KeyspaceState_ENABLED, } if err = manager.store.SaveKeyspace(defaultKeyspace); err != nil { return nil, err } - if err = manager.createNameToID(defaultKeyspaceID, defaultKeyspaceName); err != nil { - if removeErr := manager.store.RemoveKeyspace(defaultKeyspaceID); removeErr != nil { + if err = manager.createNameToID(DefaultKeyspaceID, DefaultKeyspaceName); err != nil { + if removeErr := manager.store.RemoveKeyspace(DefaultKeyspaceID); removeErr != nil { return nil, errors.Wrap(removeErr, "failed to remove keyspace meta after save spaceID failure") } return nil, err @@ -214,7 +216,7 @@ func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation) // It returns error if saving failed, operation not allowed, or if keyspace not exists. func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.KeyspaceState, now time.Time) (*keyspacepb.KeyspaceMeta, error) { // Changing the state of default keyspace is not allowed. - if name == defaultKeyspaceName { + if name == DefaultKeyspaceName { return nil, errModifyDefault } manager.metaLock.Lock() @@ -263,7 +265,7 @@ func (manager *Manager) allocID() (uint32, error) { } id32 := uint32(id64) // Skip reserved space ID. - if id32 == defaultKeyspaceID { + if id32 == DefaultKeyspaceID { return manager.allocID() } if err = validateID(id32); err != nil { diff --git a/server/keyspace/keyspace_test.go b/server/keyspace/keyspace_test.go index 11ed42e2257..97fd20464cd 100644 --- a/server/keyspace/keyspace_test.go +++ b/server/keyspace/keyspace_test.go @@ -124,7 +124,7 @@ func TestUpdateKeyspaceConfig(t *testing.T) { re.Error(err) } // Changing config of DEFAULT keyspace is allowed. - updated, err := manager.UpdateKeyspaceConfig(defaultKeyspaceName, mutations) + updated, err := manager.UpdateKeyspaceConfig(DefaultKeyspaceName, mutations) re.NoError(err) checkMutations(re, nil, updated.Config, mutations) } @@ -161,7 +161,7 @@ func TestUpdateKeyspaceState(t *testing.T) { _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ENABLED, newTime) re.Error(err) // Changing state of DEFAULT keyspace is not allowed. - _, err = manager.UpdateKeyspaceState(defaultKeyspaceName, keyspacepb.KeyspaceState_DISABLED, time.Now()) + _, err = manager.UpdateKeyspaceState(DefaultKeyspaceName, keyspacepb.KeyspaceState_DISABLED, time.Now()) re.Error(err) } } diff --git a/server/keyspace/util.go b/server/keyspace/util.go index 6cd57b08ca9..c3110c2e2e3 100644 --- a/server/keyspace/util.go +++ b/server/keyspace/util.go @@ -48,7 +48,7 @@ func validateID(spaceID uint32) error { if spaceID > spaceIDMax { return errIllegalID } - if spaceID == defaultKeyspaceID { + if spaceID == DefaultKeyspaceID { return errIllegalID } return nil @@ -65,7 +65,7 @@ func validateName(name string) error { if !isValid { return errIllegalName } - if name == defaultKeyspaceName { + if name == DefaultKeyspaceName { return errIllegalName } return nil diff --git a/server/keyspace/util_test.go b/server/keyspace/util_test.go index e4e59924e00..2d4d7aef0bf 100644 --- a/server/keyspace/util_test.go +++ b/server/keyspace/util_test.go @@ -27,7 +27,7 @@ func TestValidateID(t *testing.T) { id uint32 hasErr bool }{ - {0, true}, // Reserved id should result in error. + {DefaultKeyspaceID, true}, // Reserved id should result in error. {100, false}, {spaceIDMax - 1, false}, {spaceIDMax, false}, @@ -45,7 +45,7 @@ func TestValidateName(t *testing.T) { name string hasErr bool }{ - {"DEFAULT", true}, // Reserved name should result in error. + {DefaultKeyspaceName, true}, // Reserved name should result in error. {"keyspaceName1", false}, {"keyspace_name_1", false}, {"10", false}, From 7df25aa78caea4e9c705407e0df2a359e7283236 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 22 Jul 2022 16:45:02 +0800 Subject: [PATCH 37/76] client: use exported default keyspace Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- tests/client/keyspace_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/client/keyspace_test.go b/tests/client/keyspace_test.go index 2e8d5ec1e6d..6fde5e35ffb 100644 --- a/tests/client/keyspace_test.go +++ b/tests/client/keyspace_test.go @@ -58,10 +58,10 @@ func (suite *clientTestSuite) TestLoadKeyspace() { _, err := suite.client.LoadKeyspace(suite.ctx, "non-existing keyspace") re.Error(err) // Loading default keyspace should be successful. - keyspaceDefault, err := suite.client.LoadKeyspace(suite.ctx, "DEFAULT") + keyspaceDefault, err := suite.client.LoadKeyspace(suite.ctx, keyspace.DefaultKeyspaceName) re.NoError(err) - re.Equal(uint32(0), keyspaceDefault.Id) - re.Equal("DEFAULT", keyspaceDefault.Name) + re.Equal(keyspace.DefaultKeyspaceID, keyspaceDefault.Id) + re.Equal(keyspace.DefaultKeyspaceName, keyspaceDefault.Name) } func (suite *clientTestSuite) TestWatchKeyspace() { @@ -103,7 +103,7 @@ func (suite *clientTestSuite) TestWatchKeyspace() { loaded = <-watchChan re.Equal([]*keyspacepb.KeyspaceMeta{expected}, loaded) // Updates to default keyspace's config should also be captured. - expected, err = suite.srv.GetKeyspaceManager().UpdateKeyspaceConfig("DEFAULT", []*keyspace.Mutation{ + expected, err = suite.srv.GetKeyspaceManager().UpdateKeyspaceConfig(keyspace.DefaultKeyspaceName, []*keyspace.Mutation{ { Op: keyspace.OpPut, Key: "config", From 4a4c4db03511b59be1cab1e55b8329d8dbe626e9 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 22 Jul 2022 16:45:18 +0800 Subject: [PATCH 38/76] api: add unit test for load all Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 2 +- tests/server/apiv2/handlers/keyspace_test.go | 50 +++++++++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index 260ab495c0f..b3d50d6d0db 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -109,7 +109,7 @@ func parseLoadAllQuery(c *gin.Context) (scanStart uint32, scanLimit int, err err pageToken, set := c.GetQuery("page_token") if !set || pageToken == "" { // If pageToken is empty or unset, then scan from spaceID of 1. - scanStart = 1 + scanStart = 0 } else { scanStart64, err := strconv.ParseUint(pageToken, 10, 32) if err != nil { diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index dc61ddeaee4..fb313a215a9 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/testutil" "github.com/tikv/pd/server/apiv2/handlers" + "github.com/tikv/pd/server/keyspace" "github.com/tikv/pd/tests" "go.uber.org/goleak" "io" @@ -55,7 +56,7 @@ func TestKeyspaceTestSuite(t *testing.T) { suite.Run(t, new(keyspaceTestSuite)) } -func (suite *keyspaceTestSuite) SetupSuite() { +func (suite *keyspaceTestSuite) SetupTest() { ctx, cancel := context.WithCancel(context.Background()) suite.cleanup = cancel cluster, err := tests.NewTestCluster(ctx, 3) @@ -67,7 +68,7 @@ func (suite *keyspaceTestSuite) SetupSuite() { suite.NoError(suite.server.BootstrapCluster()) } -func (suite *keyspaceTestSuite) TearDownSuite() { +func (suite *keyspaceTestSuite) TearDownTest() { suite.cleanup() suite.cluster.Destroy() } @@ -79,11 +80,14 @@ func (suite *keyspaceTestSuite) TestCreateLoadKeyspace() { loaded := mustLoadKeyspaces(re, suite.server, created.Name) re.Equal(created, loaded) } + defaultKeyspace := mustLoadKeyspaces(re, suite.server, keyspace.DefaultKeyspaceName) + re.Equal(keyspace.DefaultKeyspaceName, defaultKeyspace.Name) + re.Equal(keyspacepb.KeyspaceState_ENABLED, defaultKeyspace.State) } func (suite *keyspaceTestSuite) TestUpdateKeyspaceConfig() { re := suite.Require() - keyspaces := mustMakeTestKeyspaces(re, suite.server, 10, 10) + keyspaces := mustMakeTestKeyspaces(re, suite.server, 0, 10) for _, created := range keyspaces { config1val := "300" updateRequest := &handlers.UpdateConfigParams{ @@ -99,7 +103,7 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceConfig() { func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { re := suite.Require() - keyspaces := mustMakeTestKeyspaces(re, suite.server, 20, 10) + keyspaces := mustMakeTestKeyspaces(re, suite.server, 0, 10) for _, created := range keyspaces { // Should NOT allow archiving ENABLED keyspace. success, _ := sendUpdateStateRequest(re, suite.server, created.Name, "archive") @@ -121,10 +125,45 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { re.False(success) } // Changing default keyspace's state is NOT allowed. - success, _ := sendUpdateStateRequest(re, suite.server, "DEFAULT", "disable") + success, _ := sendUpdateStateRequest(re, suite.server, keyspace.DefaultKeyspaceName, "disable") re.False(success) } +func (suite *keyspaceTestSuite) TestLoadRangeKeyspace() { + re := suite.Require() + keyspaces := mustMakeTestKeyspaces(re, suite.server, 0, 50) + loadResponse := sendLoadRangeRequest(re, suite.server, "", "") + re.Empty(loadResponse.NextPageToken) // Load response should contain no more pages. + // Load response should contain all created keyspace and a default. + re.Equal(len(keyspaces)+1, len(loadResponse.Keyspaces)) + for i, created := range keyspaces { + re.Equal(created, loadResponse.Keyspaces[i+1].KeyspaceMeta) + } + re.Equal(keyspace.DefaultKeyspaceName, loadResponse.Keyspaces[0].Name) + re.Equal(keyspacepb.KeyspaceState_ENABLED, loadResponse.Keyspaces[0].State) +} + +func sendLoadRangeRequest(re *require.Assertions, server *tests.TestServer, token, limit string) *handlers.LoadAllKeyspacesResponse { + // Construct load range request. + httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+keyspacesPrefix, nil) + re.NoError(err) + query := httpReq.URL.Query() + query.Add("page_token", token) + query.Add("limit", limit) + httpReq.URL.RawQuery = query.Encode() + // Send request. + httpResp, err := dialClient.Do(httpReq) + re.Equal(http.StatusOK, httpResp.StatusCode) + re.NoError(err) + // Receive & decode response. + data, err := io.ReadAll(httpResp.Body) + re.NoError(err) + re.NoError(httpResp.Body.Close()) + resp := &handlers.LoadAllKeyspacesResponse{} + re.NoError(json.Unmarshal(data, resp)) + return resp +} + func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, name, action string) (bool, *keyspacepb.KeyspaceMeta) { httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspacesPrefix+"/"+name+"/"+action, nil) re.NoError(err) @@ -150,7 +189,6 @@ func mustMakeTestKeyspaces(re *require.Assertions, server *tests.TestServer, sta Name: fmt.Sprintf("test_keyspace%d", start+i), Config: testConfig, } - fmt.Println(createRequest) resultMeta[i] = mustCreateKeyspace(re, server, createRequest) } return resultMeta From 8c61ff20f4416e9b91396826c06df1e4ea2fb0d1 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 27 Jul 2022 11:07:20 +0800 Subject: [PATCH 39/76] update comments Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/kv/etcd_kv.go | 6 +++++- server/storage/kv/kv.go | 1 + server/storage/kv/kv_test.go | 26 +++++--------------------- server/storage/kv/levedb_kv.go | 2 ++ server/storage/kv/mem_kv.go | 2 +- 5 files changed, 14 insertions(+), 23 deletions(-) diff --git a/server/storage/kv/etcd_kv.go b/server/storage/kv/etcd_kv.go index aee156988d9..9863d87dc46 100644 --- a/server/storage/kv/etcd_kv.go +++ b/server/storage/kv/etcd_kv.go @@ -174,7 +174,10 @@ func (t *SlowLogTxn) Commit() (*clientv3.TxnResponse, error) { } // etcdTxn is used to record user's action during RunInTxn, -// It stores load result in conditions and modification in operations. +// It stores modification in operations to apply as a single transaction during commit. +// All load/loadRange result will be stored in conditions. +// Transaction commit will be successful only if all conditions are met, +// aka, no other transaction has modified values loaded during current transaction. type etcdTxn struct { kv *etcdKVBase ctx context.Context @@ -198,6 +201,7 @@ func (kv *etcdKVBase) RunInTxn(ctx context.Context, f func(txn Txn) error) error } // Save puts a put operation into operations. +// Note that save result are not immediately observable before current transaction commit. func (txn *etcdTxn) Save(key, value string) error { key = path.Join(txn.kv.rootPath, key) operation := clientv3.OpPut(key, value) diff --git a/server/storage/kv/kv.go b/server/storage/kv/kv.go index f5338aebd25..a6e870db9c9 100644 --- a/server/storage/kv/kv.go +++ b/server/storage/kv/kv.go @@ -34,6 +34,7 @@ type Base interface { // returned by RunInTxn. // Otherwise, it returns the error occurred during the // transaction. + // Note that transaction are not committed until RunInTxn returns nil. // Note: // 1. Load and LoadRange operations provides only stale read. // Values saved/ removed during transaction will not be immediately diff --git a/server/storage/kv/kv_test.go b/server/storage/kv/kv_test.go index a453bf3126a..956b647fe9d 100644 --- a/server/storage/kv/kv_test.go +++ b/server/storage/kv/kv_test.go @@ -50,25 +50,6 @@ func TestEtcd(t *testing.T) { testLoadConflict(re, kv) } -func TestEtcdRunInTxn(t *testing.T) { - re := require.New(t) - cfg := newTestSingleConfig(t) - etcd, err := embed.StartEtcd(cfg) - re.NoError(err) - defer etcd.Close() - - ep := cfg.LCUrls[0].String() - client, err := clientv3.New(clientv3.Config{ - Endpoints: []string{ep}, - }) - re.NoError(err) - rootPath := path.Join("/pd", strconv.FormatUint(100, 10)) - - kv := NewEtcdKVBase(client, rootPath) - testReadWrite(re, kv) - testRange(re, kv) - testSaveMultiple(re, kv, 20) -} func TestLevelDB(t *testing.T) { re := require.New(t) dir := t.TempDir() @@ -77,6 +58,7 @@ func TestLevelDB(t *testing.T) { testReadWrite(re, kv) testRange(re, kv) + testSaveMultiple(re, kv, 20) } func TestMemKV(t *testing.T) { @@ -162,8 +144,7 @@ func newTestSingleConfig(t *testing.T) *embed.Config { } func testSaveMultiple(re *require.Assertions, kv Base, count int) { - var err error - err = kv.RunInTxn(context.Background(), func(txn Txn) error { + err := kv.RunInTxn(context.Background(), func(txn Txn) error { var saveErr error for i := 0; i < count; i++ { saveErr = txn.Save("key"+strconv.Itoa(i), "val"+strconv.Itoa(i)) @@ -181,6 +162,9 @@ func testSaveMultiple(re *require.Assertions, kv Base, count int) { } } +// testLoadConflict checks that if any value loaded during the current transaction +// has been modified by another transaction before the current one commit, +// then the current transaction must fail. func testLoadConflict(re *require.Assertions, kv Base) { re.NoError(kv.Save("testKey", "initialValue")) // loader loads the test key value. diff --git a/server/storage/kv/levedb_kv.go b/server/storage/kv/levedb_kv.go index 64207aea2d0..0b8577d399a 100644 --- a/server/storage/kv/levedb_kv.go +++ b/server/storage/kv/levedb_kv.go @@ -98,6 +98,8 @@ func (kv *LevelDBKV) SaveRegions(regions map[string]*metapb.Region) error { return nil } +// levelDBTxn implements kv.Txn. +// It utilizes leveldb.Batch to batch user operations to an atomic execution unit. type levelDBTxn struct { kv *LevelDBKV ctx context.Context diff --git a/server/storage/kv/mem_kv.go b/server/storage/kv/mem_kv.go index 9f98b23b48f..015e0154c05 100644 --- a/server/storage/kv/mem_kv.go +++ b/server/storage/kv/mem_kv.go @@ -90,7 +90,7 @@ func (kv *memoryKV) Remove(key string) error { return nil } -// memTxn implements kv.Txn +// memTxn implements kv.Txn. type memTxn struct { kv *memoryKV ctx context.Context From 5811b0d23a34557af1831aaac6cf54b73a209d5a Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 27 Jul 2022 14:02:17 +0800 Subject: [PATCH 40/76] storage: fix lint Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/keyspace.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/storage/endpoint/keyspace.go b/server/storage/endpoint/keyspace.go index 357346e867a..99aa504ddef 100644 --- a/server/storage/endpoint/keyspace.go +++ b/server/storage/endpoint/keyspace.go @@ -35,6 +35,7 @@ const ( spaceIDStrLen = 8 ) +// KeyspaceStorage defines storage operations on keyspace related data. type KeyspaceStorage interface { // SaveKeyspace saves the given keyspace to the storage. SaveKeyspace(*keyspacepb.KeyspaceMeta) error From a96fd687f417a60d9e9131d80eef1607bee44db9 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 22 Jul 2022 16:43:58 +0800 Subject: [PATCH 41/76] server: export default keyspace name and id Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 22 ++++++++++++---------- server/keyspace/keyspace_test.go | 4 ++-- server/keyspace/util.go | 4 ++-- server/keyspace/util_test.go | 4 ++-- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index ab1ec2e18a9..ec5874f3578 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -29,9 +29,11 @@ const ( // Use a lower value for denser idAllocation in the event of frequent pd leader change. AllocStep = uint64(100) // AllocLabel is used to label keyspace idAllocator's metrics. - AllocLabel = "keyspace-idAlloc" - defaultKeyspaceName = "DEFAULT" - defaultKeyspaceID = uint32(0) + AllocLabel = "keyspace-idAlloc" + // DefaultKeyspaceName is the name reserved for default keyspace. + DefaultKeyspaceName = "DEFAULT" + // DefaultKeyspaceID is the id of default keyspace. + DefaultKeyspaceID = uint32(0) ) // Manager manages keyspace related data. @@ -66,7 +68,7 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator manager.metaLock.Lock() defer manager.metaLock.Unlock() // Check if default keyspace already exists. - defaultExists, err := manager.store.LoadKeyspace(defaultKeyspaceID, &keyspacepb.KeyspaceMeta{}) + defaultExists, err := manager.store.LoadKeyspace(DefaultKeyspaceID, &keyspacepb.KeyspaceMeta{}) if err != nil { return nil, err } @@ -75,15 +77,15 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator } // Initialize default keyspace. defaultKeyspace := &keyspacepb.KeyspaceMeta{ - Id: defaultKeyspaceID, - Name: defaultKeyspaceName, + Id: DefaultKeyspaceID, + Name: DefaultKeyspaceName, State: keyspacepb.KeyspaceState_ENABLED, } if err = manager.store.SaveKeyspace(defaultKeyspace); err != nil { return nil, err } - if err = manager.createNameToID(defaultKeyspaceID, defaultKeyspaceName); err != nil { - if removeErr := manager.store.RemoveKeyspace(defaultKeyspaceID); removeErr != nil { + if err = manager.createNameToID(DefaultKeyspaceID, DefaultKeyspaceName); err != nil { + if removeErr := manager.store.RemoveKeyspace(DefaultKeyspaceID); removeErr != nil { return nil, errors.Wrap(removeErr, "failed to remove keyspace meta after save spaceID failure") } return nil, err @@ -214,7 +216,7 @@ func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation) // It returns error if saving failed, operation not allowed, or if keyspace not exists. func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.KeyspaceState, now time.Time) (*keyspacepb.KeyspaceMeta, error) { // Changing the state of default keyspace is not allowed. - if name == defaultKeyspaceName { + if name == DefaultKeyspaceName { return nil, errModifyDefault } manager.metaLock.Lock() @@ -263,7 +265,7 @@ func (manager *Manager) allocID() (uint32, error) { } id32 := uint32(id64) // Skip reserved space ID. - if id32 == defaultKeyspaceID { + if id32 == DefaultKeyspaceID { return manager.allocID() } if err = validateID(id32); err != nil { diff --git a/server/keyspace/keyspace_test.go b/server/keyspace/keyspace_test.go index 11ed42e2257..97fd20464cd 100644 --- a/server/keyspace/keyspace_test.go +++ b/server/keyspace/keyspace_test.go @@ -124,7 +124,7 @@ func TestUpdateKeyspaceConfig(t *testing.T) { re.Error(err) } // Changing config of DEFAULT keyspace is allowed. - updated, err := manager.UpdateKeyspaceConfig(defaultKeyspaceName, mutations) + updated, err := manager.UpdateKeyspaceConfig(DefaultKeyspaceName, mutations) re.NoError(err) checkMutations(re, nil, updated.Config, mutations) } @@ -161,7 +161,7 @@ func TestUpdateKeyspaceState(t *testing.T) { _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ENABLED, newTime) re.Error(err) // Changing state of DEFAULT keyspace is not allowed. - _, err = manager.UpdateKeyspaceState(defaultKeyspaceName, keyspacepb.KeyspaceState_DISABLED, time.Now()) + _, err = manager.UpdateKeyspaceState(DefaultKeyspaceName, keyspacepb.KeyspaceState_DISABLED, time.Now()) re.Error(err) } } diff --git a/server/keyspace/util.go b/server/keyspace/util.go index 6cd57b08ca9..c3110c2e2e3 100644 --- a/server/keyspace/util.go +++ b/server/keyspace/util.go @@ -48,7 +48,7 @@ func validateID(spaceID uint32) error { if spaceID > spaceIDMax { return errIllegalID } - if spaceID == defaultKeyspaceID { + if spaceID == DefaultKeyspaceID { return errIllegalID } return nil @@ -65,7 +65,7 @@ func validateName(name string) error { if !isValid { return errIllegalName } - if name == defaultKeyspaceName { + if name == DefaultKeyspaceName { return errIllegalName } return nil diff --git a/server/keyspace/util_test.go b/server/keyspace/util_test.go index e4e59924e00..2d4d7aef0bf 100644 --- a/server/keyspace/util_test.go +++ b/server/keyspace/util_test.go @@ -27,7 +27,7 @@ func TestValidateID(t *testing.T) { id uint32 hasErr bool }{ - {0, true}, // Reserved id should result in error. + {DefaultKeyspaceID, true}, // Reserved id should result in error. {100, false}, {spaceIDMax - 1, false}, {spaceIDMax, false}, @@ -45,7 +45,7 @@ func TestValidateName(t *testing.T) { name string hasErr bool }{ - {"DEFAULT", true}, // Reserved name should result in error. + {DefaultKeyspaceName, true}, // Reserved name should result in error. {"keyspaceName1", false}, {"keyspace_name_1", false}, {"10", false}, From 29da7ff617b4c4a25ad165d9c633fa13a35dcfe4 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 22 Jul 2022 16:45:02 +0800 Subject: [PATCH 42/76] client: use exported default keyspace Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- tests/client/keyspace_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/client/keyspace_test.go b/tests/client/keyspace_test.go index 2e8d5ec1e6d..6fde5e35ffb 100644 --- a/tests/client/keyspace_test.go +++ b/tests/client/keyspace_test.go @@ -58,10 +58,10 @@ func (suite *clientTestSuite) TestLoadKeyspace() { _, err := suite.client.LoadKeyspace(suite.ctx, "non-existing keyspace") re.Error(err) // Loading default keyspace should be successful. - keyspaceDefault, err := suite.client.LoadKeyspace(suite.ctx, "DEFAULT") + keyspaceDefault, err := suite.client.LoadKeyspace(suite.ctx, keyspace.DefaultKeyspaceName) re.NoError(err) - re.Equal(uint32(0), keyspaceDefault.Id) - re.Equal("DEFAULT", keyspaceDefault.Name) + re.Equal(keyspace.DefaultKeyspaceID, keyspaceDefault.Id) + re.Equal(keyspace.DefaultKeyspaceName, keyspaceDefault.Name) } func (suite *clientTestSuite) TestWatchKeyspace() { @@ -103,7 +103,7 @@ func (suite *clientTestSuite) TestWatchKeyspace() { loaded = <-watchChan re.Equal([]*keyspacepb.KeyspaceMeta{expected}, loaded) // Updates to default keyspace's config should also be captured. - expected, err = suite.srv.GetKeyspaceManager().UpdateKeyspaceConfig("DEFAULT", []*keyspace.Mutation{ + expected, err = suite.srv.GetKeyspaceManager().UpdateKeyspaceConfig(keyspace.DefaultKeyspaceName, []*keyspace.Mutation{ { Op: keyspace.OpPut, Key: "config", From 7cc5a59b3fb1deea0a1b1cde3a09e1a0e53df18b Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 22 Jul 2022 16:43:58 +0800 Subject: [PATCH 43/76] server: export default keyspace name and id Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 22 ++++++++++++---------- server/keyspace/keyspace_test.go | 4 ++-- server/keyspace/util.go | 4 ++-- server/keyspace/util_test.go | 4 ++-- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index ab1ec2e18a9..ec5874f3578 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -29,9 +29,11 @@ const ( // Use a lower value for denser idAllocation in the event of frequent pd leader change. AllocStep = uint64(100) // AllocLabel is used to label keyspace idAllocator's metrics. - AllocLabel = "keyspace-idAlloc" - defaultKeyspaceName = "DEFAULT" - defaultKeyspaceID = uint32(0) + AllocLabel = "keyspace-idAlloc" + // DefaultKeyspaceName is the name reserved for default keyspace. + DefaultKeyspaceName = "DEFAULT" + // DefaultKeyspaceID is the id of default keyspace. + DefaultKeyspaceID = uint32(0) ) // Manager manages keyspace related data. @@ -66,7 +68,7 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator manager.metaLock.Lock() defer manager.metaLock.Unlock() // Check if default keyspace already exists. - defaultExists, err := manager.store.LoadKeyspace(defaultKeyspaceID, &keyspacepb.KeyspaceMeta{}) + defaultExists, err := manager.store.LoadKeyspace(DefaultKeyspaceID, &keyspacepb.KeyspaceMeta{}) if err != nil { return nil, err } @@ -75,15 +77,15 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator } // Initialize default keyspace. defaultKeyspace := &keyspacepb.KeyspaceMeta{ - Id: defaultKeyspaceID, - Name: defaultKeyspaceName, + Id: DefaultKeyspaceID, + Name: DefaultKeyspaceName, State: keyspacepb.KeyspaceState_ENABLED, } if err = manager.store.SaveKeyspace(defaultKeyspace); err != nil { return nil, err } - if err = manager.createNameToID(defaultKeyspaceID, defaultKeyspaceName); err != nil { - if removeErr := manager.store.RemoveKeyspace(defaultKeyspaceID); removeErr != nil { + if err = manager.createNameToID(DefaultKeyspaceID, DefaultKeyspaceName); err != nil { + if removeErr := manager.store.RemoveKeyspace(DefaultKeyspaceID); removeErr != nil { return nil, errors.Wrap(removeErr, "failed to remove keyspace meta after save spaceID failure") } return nil, err @@ -214,7 +216,7 @@ func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation) // It returns error if saving failed, operation not allowed, or if keyspace not exists. func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.KeyspaceState, now time.Time) (*keyspacepb.KeyspaceMeta, error) { // Changing the state of default keyspace is not allowed. - if name == defaultKeyspaceName { + if name == DefaultKeyspaceName { return nil, errModifyDefault } manager.metaLock.Lock() @@ -263,7 +265,7 @@ func (manager *Manager) allocID() (uint32, error) { } id32 := uint32(id64) // Skip reserved space ID. - if id32 == defaultKeyspaceID { + if id32 == DefaultKeyspaceID { return manager.allocID() } if err = validateID(id32); err != nil { diff --git a/server/keyspace/keyspace_test.go b/server/keyspace/keyspace_test.go index 11ed42e2257..97fd20464cd 100644 --- a/server/keyspace/keyspace_test.go +++ b/server/keyspace/keyspace_test.go @@ -124,7 +124,7 @@ func TestUpdateKeyspaceConfig(t *testing.T) { re.Error(err) } // Changing config of DEFAULT keyspace is allowed. - updated, err := manager.UpdateKeyspaceConfig(defaultKeyspaceName, mutations) + updated, err := manager.UpdateKeyspaceConfig(DefaultKeyspaceName, mutations) re.NoError(err) checkMutations(re, nil, updated.Config, mutations) } @@ -161,7 +161,7 @@ func TestUpdateKeyspaceState(t *testing.T) { _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ENABLED, newTime) re.Error(err) // Changing state of DEFAULT keyspace is not allowed. - _, err = manager.UpdateKeyspaceState(defaultKeyspaceName, keyspacepb.KeyspaceState_DISABLED, time.Now()) + _, err = manager.UpdateKeyspaceState(DefaultKeyspaceName, keyspacepb.KeyspaceState_DISABLED, time.Now()) re.Error(err) } } diff --git a/server/keyspace/util.go b/server/keyspace/util.go index 6cd57b08ca9..c3110c2e2e3 100644 --- a/server/keyspace/util.go +++ b/server/keyspace/util.go @@ -48,7 +48,7 @@ func validateID(spaceID uint32) error { if spaceID > spaceIDMax { return errIllegalID } - if spaceID == defaultKeyspaceID { + if spaceID == DefaultKeyspaceID { return errIllegalID } return nil @@ -65,7 +65,7 @@ func validateName(name string) error { if !isValid { return errIllegalName } - if name == defaultKeyspaceName { + if name == DefaultKeyspaceName { return errIllegalName } return nil diff --git a/server/keyspace/util_test.go b/server/keyspace/util_test.go index e4e59924e00..2d4d7aef0bf 100644 --- a/server/keyspace/util_test.go +++ b/server/keyspace/util_test.go @@ -27,7 +27,7 @@ func TestValidateID(t *testing.T) { id uint32 hasErr bool }{ - {0, true}, // Reserved id should result in error. + {DefaultKeyspaceID, true}, // Reserved id should result in error. {100, false}, {spaceIDMax - 1, false}, {spaceIDMax, false}, @@ -45,7 +45,7 @@ func TestValidateName(t *testing.T) { name string hasErr bool }{ - {"DEFAULT", true}, // Reserved name should result in error. + {DefaultKeyspaceName, true}, // Reserved name should result in error. {"keyspaceName1", false}, {"keyspace_name_1", false}, {"10", false}, From 2bd8eac9586cb13e013a113ebc7312f12e246715 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 27 Jul 2022 14:02:17 +0800 Subject: [PATCH 44/76] storage: fix lint Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/keyspace.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/storage/endpoint/keyspace.go b/server/storage/endpoint/keyspace.go index 357346e867a..99aa504ddef 100644 --- a/server/storage/endpoint/keyspace.go +++ b/server/storage/endpoint/keyspace.go @@ -35,6 +35,7 @@ const ( spaceIDStrLen = 8 ) +// KeyspaceStorage defines storage operations on keyspace related data. type KeyspaceStorage interface { // SaveKeyspace saves the given keyspace to the storage. SaveKeyspace(*keyspacepb.KeyspaceMeta) error From 3a4426b01f505d4a5e5394273424e5aba4ad81d5 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 27 Jul 2022 14:02:17 +0800 Subject: [PATCH 45/76] storage: fix lint Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/keyspace.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/storage/endpoint/keyspace.go b/server/storage/endpoint/keyspace.go index 357346e867a..99aa504ddef 100644 --- a/server/storage/endpoint/keyspace.go +++ b/server/storage/endpoint/keyspace.go @@ -35,6 +35,7 @@ const ( spaceIDStrLen = 8 ) +// KeyspaceStorage defines storage operations on keyspace related data. type KeyspaceStorage interface { // SaveKeyspace saves the given keyspace to the storage. SaveKeyspace(*keyspacepb.KeyspaceMeta) error From 3fbd8e2d63d62178a4d85fd3395097b74d806ea6 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 27 Jul 2022 14:36:27 +0800 Subject: [PATCH 46/76] server: fix lint Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 8 ++++++++ server/keyspace_service.go | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index ec5874f3578..b636525e1c1 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -164,16 +164,24 @@ func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, err return keyspace, nil } +// Mutation represents a single operation to be applied on keyspace config. type Mutation struct { Op OpType Key string Value string } +// OpType defines the type of keyspace config operation. type OpType int const ( + // OpPut denotes a put operation onto the given config. + // If target key exists, it will put a new value, + // otherwise, it creates a new config entry. OpPut OpType = iota + 1 // Operation type starts at 1. + // OpDel denotes a deletion operation onto the given config. + // Note: OpDel is idempotent, deleting a non-existing key + // will not result in error. OpDel ) diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 0db09fd45b5..02052eff13b 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -70,6 +70,10 @@ func (s *KeyspaceServer) getErrorHeader(err error) *pdpb.ResponseHeader { } } +// LoadKeyspace load and return target keyspace metadata. +// Request must specify keyspace name. +// On Error, keyspaceMeta in response will be nil, +// error information will be encoded in response header with corresponding error type. func (s *KeyspaceServer) LoadKeyspace(_ context.Context, request *keyspacepb.LoadKeyspaceRequest) (*keyspacepb.LoadKeyspaceResponse, error) { rc := s.GetRaftCluster() if rc == nil { @@ -87,6 +91,8 @@ func (s *KeyspaceServer) LoadKeyspace(_ context.Context, request *keyspacepb.Loa }, nil } +// WatchKeyspaces captures and sends keyspace metadata changes to the client via gRPC stream. +// Note: It sends all existing keyspaces as it's first package to the client. func (s *KeyspaceServer) WatchKeyspaces(_ *keyspacepb.WatchKeyspacesRequest, stream keyspacepb.Keyspace_WatchKeyspacesServer) error { rc := s.GetRaftCluster() if rc == nil { From d205a8027739848d476bbf4607ed9b7a71455429 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 27 Jul 2022 14:53:47 +0800 Subject: [PATCH 47/76] fmt Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- tests/client/keyspace_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/client/keyspace_test.go b/tests/client/keyspace_test.go index 6fde5e35ffb..035d944f540 100644 --- a/tests/client/keyspace_test.go +++ b/tests/client/keyspace_test.go @@ -46,6 +46,7 @@ func mustMakeTestKeyspaces(re *require.Assertions, server *server.Server, start, } return keyspaces } + func (suite *clientTestSuite) TestLoadKeyspace() { re := suite.Require() metas := mustMakeTestKeyspaces(re, suite.srv, 0, 10) From 04961f7544c5bf43b720692e87677fdcb3aa6b26 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 27 Jul 2022 15:39:11 +0800 Subject: [PATCH 48/76] fix lint Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 11 +++++++---- tests/server/apiv2/handlers/keyspace_test.go | 15 +++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index b3d50d6d0db..39e0347f0a3 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -28,6 +28,7 @@ import ( "github.com/tikv/pd/server/keyspace" ) +// RegisterKeyspace register keyspace related handlers to router paths. func RegisterKeyspace(r *gin.RouterGroup) { router := r.Group("keyspaces") router.Use(middlewares.BootstrapChecker()) @@ -134,6 +135,8 @@ func parseLoadAllQuery(c *gin.Context) (scanStart uint32, scanLimit int, err err return scanStart, scanLimit, nil } +// LoadAllKeyspacesResponse represents response given when loading all keyspaces. +// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. type LoadAllKeyspacesResponse struct { Keyspaces []*KeyspaceMeta `json:"keyspaces"` // Token that can be used to read immediate next page. @@ -254,7 +257,7 @@ func getMutations(patch map[string]*string) []*keyspace.Mutation { // @Failure 500 {string} string "PD server failed to proceed the request." // Router /keyspaces/{name}/enable [post] func EnableKeyspace(c *gin.Context) { - UpdateKeyspaceState(c, keyspacepb.KeyspaceState_ENABLED) + updateKeyspaceState(c, keyspacepb.KeyspaceState_ENABLED) } // DisableKeyspace disables target keyspace. @@ -266,7 +269,7 @@ func EnableKeyspace(c *gin.Context) { // @Failure 500 {string} string "PD server failed to proceed the request." // Router /keyspaces/{name}/disable [post] func DisableKeyspace(c *gin.Context) { - UpdateKeyspaceState(c, keyspacepb.KeyspaceState_DISABLED) + updateKeyspaceState(c, keyspacepb.KeyspaceState_DISABLED) } // ArchiveKeyspace archives target keyspace. @@ -278,9 +281,9 @@ func DisableKeyspace(c *gin.Context) { // @Failure 500 {string} string "PD server failed to proceed the request." // Router /keyspaces/{name}/archive [post] func ArchiveKeyspace(c *gin.Context) { - UpdateKeyspaceState(c, keyspacepb.KeyspaceState_ARCHIVED) + updateKeyspaceState(c, keyspacepb.KeyspaceState_ARCHIVED) } -func UpdateKeyspaceState(c *gin.Context, state keyspacepb.KeyspaceState) { +func updateKeyspaceState(c *gin.Context, state keyspacepb.KeyspaceState) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() name := c.Param("name") diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index fb313a215a9..e0a46c10b48 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -75,7 +75,7 @@ func (suite *keyspaceTestSuite) TearDownTest() { func (suite *keyspaceTestSuite) TestCreateLoadKeyspace() { re := suite.Require() - keyspaces := mustMakeTestKeyspaces(re, suite.server, 0, 10) + keyspaces := mustMakeTestKeyspaces(re, suite.server, 10) for _, created := range keyspaces { loaded := mustLoadKeyspaces(re, suite.server, created.Name) re.Equal(created, loaded) @@ -87,7 +87,7 @@ func (suite *keyspaceTestSuite) TestCreateLoadKeyspace() { func (suite *keyspaceTestSuite) TestUpdateKeyspaceConfig() { re := suite.Require() - keyspaces := mustMakeTestKeyspaces(re, suite.server, 0, 10) + keyspaces := mustMakeTestKeyspaces(re, suite.server, 10) for _, created := range keyspaces { config1val := "300" updateRequest := &handlers.UpdateConfigParams{ @@ -103,7 +103,7 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceConfig() { func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { re := suite.Require() - keyspaces := mustMakeTestKeyspaces(re, suite.server, 0, 10) + keyspaces := mustMakeTestKeyspaces(re, suite.server, 10) for _, created := range keyspaces { // Should NOT allow archiving ENABLED keyspace. success, _ := sendUpdateStateRequest(re, suite.server, created.Name, "archive") @@ -131,7 +131,7 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { func (suite *keyspaceTestSuite) TestLoadRangeKeyspace() { re := suite.Require() - keyspaces := mustMakeTestKeyspaces(re, suite.server, 0, 50) + keyspaces := mustMakeTestKeyspaces(re, suite.server, 50) loadResponse := sendLoadRangeRequest(re, suite.server, "", "") re.Empty(loadResponse.NextPageToken) // Load response should contain no more pages. // Load response should contain all created keyspace and a default. @@ -168,6 +168,7 @@ func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, na httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspacesPrefix+"/"+name+"/"+action, nil) re.NoError(err) resp, err := dialClient.Do(httpReq) + re.NoError(err) if resp.StatusCode != http.StatusOK { return false, nil } @@ -178,7 +179,7 @@ func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, na re.NoError(json.Unmarshal(data, meta)) return true, meta.KeyspaceMeta } -func mustMakeTestKeyspaces(re *require.Assertions, server *tests.TestServer, start, count int) []*keyspacepb.KeyspaceMeta { +func mustMakeTestKeyspaces(re *require.Assertions, server *tests.TestServer, count int) []*keyspacepb.KeyspaceMeta { testConfig := map[string]string{ "config1": "100", "config2": "200", @@ -186,7 +187,7 @@ func mustMakeTestKeyspaces(re *require.Assertions, server *tests.TestServer, sta resultMeta := make([]*keyspacepb.KeyspaceMeta, count) for i := 0; i < count; i++ { createRequest := &handlers.CreateKeyspaceParams{ - Name: fmt.Sprintf("test_keyspace%d", start+i), + Name: fmt.Sprintf("test_keyspace%d", i), Config: testConfig, } resultMeta[i] = mustCreateKeyspace(re, server, createRequest) @@ -198,6 +199,7 @@ func mustCreateKeyspace(re *require.Assertions, server *tests.TestServer, reques data, err := json.Marshal(request) re.NoError(err) httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspacesPrefix, bytes.NewBuffer(data)) + re.NoError(err) resp, err := dialClient.Do(httpReq) re.NoError(err) re.Equal(http.StatusOK, resp.StatusCode) @@ -214,6 +216,7 @@ func mustUpdateKeyspaceConfig(re *require.Assertions, server *tests.TestServer, data, err := json.Marshal(request) re.NoError(err) httpReq, err := http.NewRequest(http.MethodPatch, server.GetAddr()+keyspacesPrefix+"/"+name+"/updateConfig", bytes.NewBuffer(data)) + re.NoError(err) resp, err := dialClient.Do(httpReq) re.NoError(err) re.Equal(http.StatusOK, resp.StatusCode) From a6f2521278a5f297aff8e6c464baeb5d9cf2a606 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 2 Aug 2022 11:40:27 +0800 Subject: [PATCH 49/76] change update config to follow apiv2 standard Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 6 ++++-- tests/server/apiv2/handlers/keyspace_test.go | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index 39e0347f0a3..6c561f79c95 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -35,7 +35,7 @@ func RegisterKeyspace(r *gin.RouterGroup) { router.POST("", CreateKeyspace) router.GET("", LoadAllKeyspaces) router.GET("/:name", LoadKeyspace) - router.PATCH("/:name/updateConfig", UpdateKeyspaceConfig) + router.POST("/:name/update-config", UpdateKeyspaceConfig) router.POST("/:name/enable", EnableKeyspace) router.POST("/:name/disable", DisableKeyspace) router.POST("/:name/archive", ArchiveKeyspace) @@ -201,6 +201,8 @@ type UpdateConfigParams struct { } // UpdateKeyspaceConfig updates target keyspace's config. +// This is a custom action that supports PATCH semantics and uses the JSON merge patch +// format and processing rules. // @Tags keyspaces // @Summary Update keyspace config. // @Param name path string true "Keyspace Name" @@ -208,7 +210,7 @@ type UpdateConfigParams struct { // @Produce json // @Success 200 {object} KeyspaceMeta // @Failure 500 {string} string "PD server failed to proceed the request." -// Router /keyspaces/{name}/updateConfig [patch] +// Router /keyspaces/{name}/update-config [post] func UpdateKeyspaceConfig(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index e0a46c10b48..3278de9fc62 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -215,7 +215,7 @@ func mustCreateKeyspace(re *require.Assertions, server *tests.TestServer, reques func mustUpdateKeyspaceConfig(re *require.Assertions, server *tests.TestServer, name string, request *handlers.UpdateConfigParams) *keyspacepb.KeyspaceMeta { data, err := json.Marshal(request) re.NoError(err) - httpReq, err := http.NewRequest(http.MethodPatch, server.GetAddr()+keyspacesPrefix+"/"+name+"/updateConfig", bytes.NewBuffer(data)) + httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspacesPrefix+"/"+name+"/update-config", bytes.NewBuffer(data)) re.NoError(err) resp, err := dialClient.Do(httpReq) re.NoError(err) From 7d8760c0cc90f7f6e7bd03dccb17edb037285603 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 5 Aug 2022 15:26:42 +0800 Subject: [PATCH 50/76] use lock group to control keyspace concurrency Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 71 +++++++++++++++++++++++------------- server/keyspace/util.go | 52 ++++++++++++++++++++++++++ server/keyspace/util_test.go | 37 +++++++++++++++++++ 3 files changed, 134 insertions(+), 26 deletions(-) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index b636525e1c1..12b74c418c2 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -42,7 +42,7 @@ type Manager struct { // idLock guards keyspace name to id lookup entries. idLock syncutil.Mutex // metaLock guards keyspace meta. - metaLock syncutil.Mutex + metaLock *lockGroup // idAllocator allocates keyspace id. idAllocator id.Allocator // store is the storage for keyspace related information. @@ -63,10 +63,9 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator manager := &Manager{ store: store, idAllocator: idAllocator, + metaLock: newLockGroup(), } - manager.metaLock.Lock() - defer manager.metaLock.Unlock() // Check if default keyspace already exists. defaultExists, err := manager.store.LoadKeyspace(DefaultKeyspaceID, &keyspacepb.KeyspaceMeta{}) if err != nil { @@ -115,9 +114,19 @@ func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspac StateChangedAt: request.Now.Unix(), Config: request.Config, } - manager.metaLock.Lock() - defer manager.metaLock.Unlock() - // Check if keyspace with that id already exists. + manager.idLock.Lock() + defer manager.idLock.Unlock() + // Check if keyspace id with that name already exists. + nameExists, _, err := manager.store.LoadKeyspaceIDByName(request.Name) + if err != nil { + return nil, err + } + if nameExists { + return nil, ErrKeyspaceExists + } + manager.metaLock.lock(newID) + defer manager.metaLock.unlock(newID) + // Check if keyspace meta with that id already exists. keyspaceExists, err := manager.store.LoadKeyspace(newID, &keyspacepb.KeyspaceMeta{}) if err != nil { return nil, err @@ -152,9 +161,13 @@ func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, err if !loaded { return nil, ErrKeyspaceNotFound } + return manager.loadKeyspaceByID(spaceID) +} + +func (manager *Manager) loadKeyspaceByID(spaceID uint32) (*keyspacepb.KeyspaceMeta, error) { // Load the keyspace with target ID. keyspace := &keyspacepb.KeyspaceMeta{} - loaded, err = manager.store.LoadKeyspace(spaceID, keyspace) + loaded, err := manager.store.LoadKeyspace(spaceID, keyspace) if err != nil { return nil, err } @@ -188,10 +201,18 @@ const ( // UpdateKeyspaceConfig changes target keyspace's config in the order specified in mutations. // It returns error if saving failed, operation not allowed, or if keyspace not exists. func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation) (*keyspacepb.KeyspaceMeta, error) { - manager.metaLock.Lock() - defer manager.metaLock.Unlock() - // Load keyspace by name. - keyspace, err := manager.LoadKeyspace(name) + // First get KeyspaceID from Name. + loaded, spaceID, err := manager.store.LoadKeyspaceIDByName(name) + if err != nil { + return nil, err + } + if !loaded { + return nil, ErrKeyspaceNotFound + } + manager.metaLock.lock(spaceID) + defer manager.metaLock.unlock(spaceID) + // Load keyspace by id. + keyspace, err := manager.loadKeyspaceByID(spaceID) if err != nil { return nil, err } @@ -227,10 +248,18 @@ func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.Key if name == DefaultKeyspaceName { return nil, errModifyDefault } - manager.metaLock.Lock() - defer manager.metaLock.Unlock() - // Load keyspace by name. - keyspace, err := manager.LoadKeyspace(name) + // First get KeyspaceID from Name. + loaded, spaceID, err := manager.store.LoadKeyspaceIDByName(name) + if err != nil { + return nil, err + } + if !loaded { + return nil, ErrKeyspaceNotFound + } + manager.metaLock.lock(spaceID) + defer manager.metaLock.unlock(spaceID) + // Load keyspace by id. + keyspace, err := manager.loadKeyspaceByID(spaceID) if err != nil { return nil, err } @@ -283,17 +312,7 @@ func (manager *Manager) allocID() (uint32, error) { } // createNameToID create a keyspace name to ID lookup entry. -// It returns error if saving keyspace name meet error or if name has already been taken. +// It returns error if saving keyspace name meet error. func (manager *Manager) createNameToID(spaceID uint32, name string) error { - manager.idLock.Lock() - defer manager.idLock.Unlock() - - nameExists, _, err := manager.store.LoadKeyspaceIDByName(name) - if err != nil { - return err - } - if nameExists { - return ErrKeyspaceExists - } return manager.store.SaveKeyspaceIDByName(spaceID, name) } diff --git a/server/keyspace/util.go b/server/keyspace/util.go index c3110c2e2e3..f19c2c97678 100644 --- a/server/keyspace/util.go +++ b/server/keyspace/util.go @@ -15,9 +15,11 @@ package keyspace import ( + "fmt" "regexp" "github.com/pingcap/errors" + "github.com/tikv/pd/pkg/syncutil" ) const ( @@ -70,3 +72,53 @@ func validateName(name string) error { } return nil } + +// lockGroup is a map of mutex that locks each keyspace separately. +// It's used to guarantee that operations on different keyspace won't block each other. +type lockGroup struct { + groupLock syncutil.Mutex // protects group. + entries map[uint32]*syncutil.Mutex // map of locks with keyspaceID as key. + // hashFn hashes spaceID to map key, it's main purpose is to limit the total + // number of mutexes in the group, as using a mutex for every keyspace is too memory heavy. + hashFn func(spaceID uint32) uint32 +} + +// newLockGroup create and return a empty lockGroup. +func newLockGroup() *lockGroup { + return &lockGroup{ + entries: make(map[uint32]*syncutil.Mutex), + // A simple mask is applied to spaceID to use its last byte as map key, + // limiting the maximum map length to 256. + // Since keyspaceID is sequentially allocated, this can also reduce the chance + // of collision when comparing with random hashes. + hashFn: func(spaceID uint32) uint32 { + return spaceID & 0xFF + }, + } +} + +func (g *lockGroup) lock(spaceID uint32) { + g.groupLock.Lock() + hashedID := spaceID & 0xFF + e, ok := g.entries[hashedID] + // If target keyspace's lock has not been initialized. + if !ok { + e = &syncutil.Mutex{} + g.entries[hashedID] = e + } + g.groupLock.Unlock() + e.Lock() +} + +func (g *lockGroup) unlock(spaceID uint32) { + g.groupLock.Lock() + hashedID := spaceID & 0xFF + e, ok := g.entries[hashedID] + if !ok { + // Entry must exist, otherwise there should be a run-time error and panic. + g.groupLock.Unlock() + panic(fmt.Errorf("unlock requested for key %v, but no entry found", spaceID)) + } + g.groupLock.Unlock() + e.Unlock() +} diff --git a/server/keyspace/util_test.go b/server/keyspace/util_test.go index 2d4d7aef0bf..a82d6328501 100644 --- a/server/keyspace/util_test.go +++ b/server/keyspace/util_test.go @@ -16,6 +16,8 @@ package keyspace import ( "math" + "math/rand" + "sync" "testing" "github.com/stretchr/testify/require" @@ -60,3 +62,38 @@ func TestValidateName(t *testing.T) { re.Equal(testCase.hasErr, validateName(testCase.name) != nil) } } + +func TestLockGroup(t *testing.T) { + re := require.New(t) + group := newLockGroup() + concurrency := 2000 + var wg sync.WaitGroup + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { + go func(spaceID uint32) { + defer wg.Done() + mustSequentialUpdateSingle(re, spaceID, group) + }(rand.Uint32()) + } + wg.Wait() + // Check that size of the lock group is limited. + re.LessOrEqual(len(group.entries), 1<<8) +} + +// mustSequentialUpdateSingle checks that for any given update, update is sequential. +func mustSequentialUpdateSingle(re *require.Assertions, spaceID uint32, group *lockGroup) { + concurrency := 1000 + total := 0 + var wg sync.WaitGroup + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { + go func() { + defer wg.Done() + group.lock(spaceID) + defer group.unlock(spaceID) + total++ + }() + } + wg.Wait() + re.Equal(concurrency, total) +} From 26990e401e344905d7a95a8285b26fa72d773647 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 8 Aug 2022 15:51:22 +0800 Subject: [PATCH 51/76] typo fix Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/key_path.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index 2f689d03497..9de751c3f46 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -161,7 +161,7 @@ func KeyspaceIDPath(name string) string { } // KeyspaceIDAlloc returns the path of the keyspace id's persistent window boundary. -// Path: keyspace/alloc_id +// Path: keyspaces/alloc_id func KeyspaceIDAlloc() string { return path.Join(keyspacePrefix, keyspaceAllocID) } From 834116bb06d1fbcee265e10cb199f3a956fd6fc1 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 8 Aug 2022 15:56:55 +0800 Subject: [PATCH 52/76] lower lockgroup test concurrency for ci test Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/util_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/keyspace/util_test.go b/server/keyspace/util_test.go index a82d6328501..1308e36c4cb 100644 --- a/server/keyspace/util_test.go +++ b/server/keyspace/util_test.go @@ -66,7 +66,7 @@ func TestValidateName(t *testing.T) { func TestLockGroup(t *testing.T) { re := require.New(t) group := newLockGroup() - concurrency := 2000 + concurrency := 20 var wg sync.WaitGroup wg.Add(concurrency) for i := 0; i < concurrency; i++ { @@ -82,7 +82,7 @@ func TestLockGroup(t *testing.T) { // mustSequentialUpdateSingle checks that for any given update, update is sequential. func mustSequentialUpdateSingle(re *require.Assertions, spaceID uint32, group *lockGroup) { - concurrency := 1000 + concurrency := 100 total := 0 var wg sync.WaitGroup wg.Add(concurrency) From fab78867a33ecda4eeab9876c660b2d4a548f4d0 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 8 Aug 2022 18:25:31 +0800 Subject: [PATCH 53/76] address comments Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- pkg/syncutil/lock_group.go | 69 +++++++++++++++++++++++++++++++++ pkg/syncutil/lock_group_test.go | 58 +++++++++++++++++++++++++++ server/keyspace/keyspace.go | 68 +++++++++++++------------------- server/keyspace/util.go | 62 ++--------------------------- server/keyspace/util_test.go | 37 ------------------ 5 files changed, 159 insertions(+), 135 deletions(-) create mode 100644 pkg/syncutil/lock_group.go create mode 100644 pkg/syncutil/lock_group_test.go diff --git a/pkg/syncutil/lock_group.go b/pkg/syncutil/lock_group.go new file mode 100644 index 00000000000..1a92d164a8a --- /dev/null +++ b/pkg/syncutil/lock_group.go @@ -0,0 +1,69 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package syncutil + +import "fmt" + +// LockGroup is a map of mutex that locks each keyspace separately. +// It's used to guarantee that operations on different keyspace won't block each other. +type LockGroup struct { + groupLock Mutex // protects group. + entries map[uint32]*Mutex // map of locks with keyspaceID as key. + // hashFn hashes spaceID to map key, it's main purpose is to limit the total + // number of mutexes in the group, as using a mutex for every keyspace is too memory heavy. + hashFn func(spaceID uint32) uint32 +} + +// NewLockGroup create and return an empty lockGroup. +func NewLockGroup() *LockGroup { + return &LockGroup{ + entries: make(map[uint32]*Mutex), + // A simple mask is applied to spaceID to use its last byte as map key, + // limiting the maximum map length to 256. + // Since keyspaceID is sequentially allocated, this can also reduce the chance + // of collision when comparing with random hashes. + hashFn: func(spaceID uint32) uint32 { + return spaceID & 0xFF + }, + } +} + +// Lock locks the given keyspace based on the hash of the spaceID. +func (g *LockGroup) Lock(spaceID uint32) { + g.groupLock.Lock() + hashedID := spaceID & 0xFF + e, ok := g.entries[hashedID] + // If target keyspace's lock has not been initialized, create a new lock. + if !ok { + e = &Mutex{} + g.entries[hashedID] = e + } + g.groupLock.Unlock() + e.Lock() +} + +// Unlock unlocks the keyspace based on the hash of the spaceID. +func (g *LockGroup) Unlock(spaceID uint32) { + g.groupLock.Lock() + hashedID := spaceID & 0xFF + e, ok := g.entries[hashedID] + if !ok { + // Entry must exist, otherwise there should be a run-time error and panic. + g.groupLock.Unlock() + panic(fmt.Errorf("unlock requested for key %v, but no entry found", spaceID)) + } + g.groupLock.Unlock() + e.Unlock() +} diff --git a/pkg/syncutil/lock_group_test.go b/pkg/syncutil/lock_group_test.go new file mode 100644 index 00000000000..a33c1deb1a6 --- /dev/null +++ b/pkg/syncutil/lock_group_test.go @@ -0,0 +1,58 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package syncutil + +import ( + "math/rand" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLockGroup(t *testing.T) { + re := require.New(t) + group := NewLockGroup() + concurrency := 20 + var wg sync.WaitGroup + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { + go func(spaceID uint32) { + defer wg.Done() + mustSequentialUpdateSingle(re, spaceID, group) + }(rand.Uint32()) + } + wg.Wait() + // Check that size of the lock group is limited. + re.LessOrEqual(len(group.entries), 1<<8) +} + +// mustSequentialUpdateSingle checks that for any given update, update is sequential. +func mustSequentialUpdateSingle(re *require.Assertions, spaceID uint32, group *LockGroup) { + concurrency := 100 + total := 0 + var wg sync.WaitGroup + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { + go func() { + defer wg.Done() + group.Lock(spaceID) + defer group.Unlock(spaceID) + total++ + }() + } + wg.Wait() + re.Equal(concurrency, total) +} diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index 12b74c418c2..8e2e73406a8 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -42,7 +42,7 @@ type Manager struct { // idLock guards keyspace name to id lookup entries. idLock syncutil.Mutex // metaLock guards keyspace meta. - metaLock *lockGroup + metaLock *syncutil.LockGroup // idAllocator allocates keyspace id. idAllocator id.Allocator // store is the storage for keyspace related information. @@ -63,33 +63,21 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator manager := &Manager{ store: store, idAllocator: idAllocator, - metaLock: newLockGroup(), - } - - // Check if default keyspace already exists. - defaultExists, err := manager.store.LoadKeyspace(DefaultKeyspaceID, &keyspacepb.KeyspaceMeta{}) - if err != nil { - return nil, err - } - if defaultExists { - return manager, nil + metaLock: syncutil.NewLockGroup(), } + now := time.Now() // Initialize default keyspace. defaultKeyspace := &keyspacepb.KeyspaceMeta{ - Id: DefaultKeyspaceID, - Name: DefaultKeyspaceName, - State: keyspacepb.KeyspaceState_ENABLED, - } - if err = manager.store.SaveKeyspace(defaultKeyspace); err != nil { - return nil, err + Id: DefaultKeyspaceID, + Name: DefaultKeyspaceName, + State: keyspacepb.KeyspaceState_ENABLED, + CreatedAt: now.Unix(), + StateChangedAt: now.Unix(), } - if err = manager.createNameToID(DefaultKeyspaceID, DefaultKeyspaceName); err != nil { - if removeErr := manager.store.RemoveKeyspace(DefaultKeyspaceID); removeErr != nil { - return nil, errors.Wrap(removeErr, "failed to remove keyspace meta after save spaceID failure") - } + _, err := manager.saveNewKeyspace(defaultKeyspace) + if err != nil { return nil, err } - return manager, nil } @@ -104,7 +92,6 @@ func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspac if err != nil { return nil, err } - // TODO: Enable Transaction at storage layer to save MetaData and NameToID in a single transaction. // Create and save keyspace metadata. keyspace := &keyspacepb.KeyspaceMeta{ Id: newID, @@ -114,35 +101,40 @@ func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspac StateChangedAt: request.Now.Unix(), Config: request.Config, } + return manager.saveNewKeyspace(keyspace) +} + +func (manager *Manager) saveNewKeyspace(keyspace *keyspacepb.KeyspaceMeta) (*keyspacepb.KeyspaceMeta, error) { manager.idLock.Lock() defer manager.idLock.Unlock() // Check if keyspace id with that name already exists. - nameExists, _, err := manager.store.LoadKeyspaceIDByName(request.Name) + nameExists, _, err := manager.store.LoadKeyspaceIDByName(keyspace.Name) if err != nil { return nil, err } if nameExists { return nil, ErrKeyspaceExists } - manager.metaLock.lock(newID) - defer manager.metaLock.unlock(newID) + manager.metaLock.Lock(keyspace.Id) + defer manager.metaLock.Unlock(keyspace.Id) // Check if keyspace meta with that id already exists. - keyspaceExists, err := manager.store.LoadKeyspace(newID, &keyspacepb.KeyspaceMeta{}) + keyspaceExists, err := manager.store.LoadKeyspace(keyspace.Id, &keyspacepb.KeyspaceMeta{}) if err != nil { return nil, err } if keyspaceExists { return nil, ErrKeyspaceExists } - // Save keyspace meta before saving id. + // TODO: Enable Transaction at storage layer to save MetaData and NameToID in a single transaction. + // Save keyspace keyspace before saving id. if err = manager.store.SaveKeyspace(keyspace); err != nil { return nil, err } // Create name to ID entry, // if this failed, previously stored keyspace meta should be removed. - if err = manager.createNameToID(newID, request.Name); err != nil { - if removeErr := manager.store.RemoveKeyspace(newID); removeErr != nil { - return nil, errors.Wrap(removeErr, "failed to remove keyspace meta after save spaceID failure") + if err = manager.createNameToID(keyspace.Id, keyspace.Name); err != nil { + if removeErr := manager.store.RemoveKeyspace(keyspace.Id); removeErr != nil { + return nil, errors.Wrap(removeErr, "failed to remove keyspace keyspace after save spaceID failure") } return nil, err } @@ -209,8 +201,8 @@ func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation) if !loaded { return nil, ErrKeyspaceNotFound } - manager.metaLock.lock(spaceID) - defer manager.metaLock.unlock(spaceID) + manager.metaLock.Lock(spaceID) + defer manager.metaLock.Unlock(spaceID) // Load keyspace by id. keyspace, err := manager.loadKeyspaceByID(spaceID) if err != nil { @@ -256,8 +248,8 @@ func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.Key if !loaded { return nil, ErrKeyspaceNotFound } - manager.metaLock.lock(spaceID) - defer manager.metaLock.unlock(spaceID) + manager.metaLock.Lock(spaceID) + defer manager.metaLock.Unlock(spaceID) // Load keyspace by id. keyspace, err := manager.loadKeyspaceByID(spaceID) if err != nil { @@ -289,7 +281,7 @@ func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.Key func (manager *Manager) LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { // Load Start should fall within acceptable ID range. if startID > spaceIDMax { - return nil, errIllegalID + return nil, errors.Errorf("startID of the scan %d exceeds spaceID Max %d", startID, spaceIDMax) } return manager.store.LoadRangeKeyspace(startID, limit) } @@ -301,10 +293,6 @@ func (manager *Manager) allocID() (uint32, error) { return 0, err } id32 := uint32(id64) - // Skip reserved space ID. - if id32 == DefaultKeyspaceID { - return manager.allocID() - } if err = validateID(id32); err != nil { return 0, err } diff --git a/server/keyspace/util.go b/server/keyspace/util.go index f19c2c97678..5c6072a84fb 100644 --- a/server/keyspace/util.go +++ b/server/keyspace/util.go @@ -15,11 +15,9 @@ package keyspace import ( - "fmt" "regexp" "github.com/pingcap/errors" - "github.com/tikv/pd/pkg/syncutil" ) const ( @@ -38,8 +36,6 @@ var ( errKeyspaceArchived = errors.New("keyspace already archived") errArchiveEnabled = errors.New("cannot archive ENABLED keyspace") errModifyDefault = errors.New("cannot modify default keyspace's state") - errIllegalID = errors.New("illegal keyspace ID") - errIllegalName = errors.New("illegal keyspace name") errIllegalOperation = errors.New("unknown operation") ) @@ -48,10 +44,10 @@ var ( // or if it collides with reserved id. func validateID(spaceID uint32) error { if spaceID > spaceIDMax { - return errIllegalID + return errors.Errorf("illegal keyspace id %d, larger than spaceID Max %d", spaceID, spaceIDMax) } if spaceID == DefaultKeyspaceID { - return errIllegalID + return errors.Errorf("illegal keyspace id %d, collides with default keyspace id", spaceID) } return nil } @@ -65,60 +61,10 @@ func validateName(name string) error { return err } if !isValid { - return errIllegalName + return errors.Errorf("illegal keyspace name %s, should contain only alphanumerical and underline", name) } if name == DefaultKeyspaceName { - return errIllegalName + return errors.Errorf("illegal keyspace name %s, collides with default keyspace name", name) } return nil } - -// lockGroup is a map of mutex that locks each keyspace separately. -// It's used to guarantee that operations on different keyspace won't block each other. -type lockGroup struct { - groupLock syncutil.Mutex // protects group. - entries map[uint32]*syncutil.Mutex // map of locks with keyspaceID as key. - // hashFn hashes spaceID to map key, it's main purpose is to limit the total - // number of mutexes in the group, as using a mutex for every keyspace is too memory heavy. - hashFn func(spaceID uint32) uint32 -} - -// newLockGroup create and return a empty lockGroup. -func newLockGroup() *lockGroup { - return &lockGroup{ - entries: make(map[uint32]*syncutil.Mutex), - // A simple mask is applied to spaceID to use its last byte as map key, - // limiting the maximum map length to 256. - // Since keyspaceID is sequentially allocated, this can also reduce the chance - // of collision when comparing with random hashes. - hashFn: func(spaceID uint32) uint32 { - return spaceID & 0xFF - }, - } -} - -func (g *lockGroup) lock(spaceID uint32) { - g.groupLock.Lock() - hashedID := spaceID & 0xFF - e, ok := g.entries[hashedID] - // If target keyspace's lock has not been initialized. - if !ok { - e = &syncutil.Mutex{} - g.entries[hashedID] = e - } - g.groupLock.Unlock() - e.Lock() -} - -func (g *lockGroup) unlock(spaceID uint32) { - g.groupLock.Lock() - hashedID := spaceID & 0xFF - e, ok := g.entries[hashedID] - if !ok { - // Entry must exist, otherwise there should be a run-time error and panic. - g.groupLock.Unlock() - panic(fmt.Errorf("unlock requested for key %v, but no entry found", spaceID)) - } - g.groupLock.Unlock() - e.Unlock() -} diff --git a/server/keyspace/util_test.go b/server/keyspace/util_test.go index 1308e36c4cb..2d4d7aef0bf 100644 --- a/server/keyspace/util_test.go +++ b/server/keyspace/util_test.go @@ -16,8 +16,6 @@ package keyspace import ( "math" - "math/rand" - "sync" "testing" "github.com/stretchr/testify/require" @@ -62,38 +60,3 @@ func TestValidateName(t *testing.T) { re.Equal(testCase.hasErr, validateName(testCase.name) != nil) } } - -func TestLockGroup(t *testing.T) { - re := require.New(t) - group := newLockGroup() - concurrency := 20 - var wg sync.WaitGroup - wg.Add(concurrency) - for i := 0; i < concurrency; i++ { - go func(spaceID uint32) { - defer wg.Done() - mustSequentialUpdateSingle(re, spaceID, group) - }(rand.Uint32()) - } - wg.Wait() - // Check that size of the lock group is limited. - re.LessOrEqual(len(group.entries), 1<<8) -} - -// mustSequentialUpdateSingle checks that for any given update, update is sequential. -func mustSequentialUpdateSingle(re *require.Assertions, spaceID uint32, group *lockGroup) { - concurrency := 100 - total := 0 - var wg sync.WaitGroup - wg.Add(concurrency) - for i := 0; i < concurrency; i++ { - go func() { - defer wg.Done() - group.lock(spaceID) - defer group.unlock(spaceID) - total++ - }() - } - wg.Wait() - re.Equal(concurrency, total) -} From f4d66f17a024cfb8d45169ac67bb22abb6150ce6 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 9 Aug 2022 13:47:42 +0800 Subject: [PATCH 54/76] let keyspace service wrap GrpcService, add cluster id check Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace_service.go | 58 ++++++++++++++++---------------------- server/server.go | 5 ++-- 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 02052eff13b..9ae280abba6 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -24,57 +24,46 @@ import ( "github.com/tikv/pd/server/keyspace" "github.com/tikv/pd/server/storage/endpoint" "go.etcd.io/etcd/clientv3" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -// KeyspaceServer wraps Server to provide keyspace service. +// KeyspaceServer wraps GrpcServer to provide keyspace service. type KeyspaceServer struct { - *Server -} - -func (s *KeyspaceServer) header() *pdpb.ResponseHeader { - return &pdpb.ResponseHeader{ClusterId: s.clusterID} -} - -func (s *KeyspaceServer) errorHeader(err *pdpb.Error) *pdpb.ResponseHeader { - return &pdpb.ResponseHeader{ - ClusterId: s.clusterID, - Error: err, - } -} - -func (s *KeyspaceServer) notBootstrappedHeader() *pdpb.ResponseHeader { - return s.errorHeader(&pdpb.Error{ - Type: pdpb.ErrorType_NOT_BOOTSTRAPPED, - Message: "cluster is not bootstrapped", - }) + *GrpcServer } // getErrorHeader returns corresponding ResponseHeader based on err. func (s *KeyspaceServer) getErrorHeader(err error) *pdpb.ResponseHeader { switch err { case keyspace.ErrKeyspaceExists: - return s.errorHeader(&pdpb.Error{ - Type: pdpb.ErrorType_DUPLICATED_ENTRY, - Message: err.Error(), - }) + return s.wrapErrorToHeader(pdpb.ErrorType_DUPLICATED_ENTRY, err.Error()) case keyspace.ErrKeyspaceNotFound: - return s.errorHeader(&pdpb.Error{ - Type: pdpb.ErrorType_ENTRY_NOT_FOUND, - Message: err.Error(), - }) + return s.wrapErrorToHeader(pdpb.ErrorType_ENTRY_NOT_FOUND, err.Error()) default: - return s.errorHeader(&pdpb.Error{ - Type: pdpb.ErrorType_UNKNOWN, - Message: err.Error(), - }) + return s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()) } } +// validateRequest checks that server is serving and cluster id matches the requested. +func (s *KeyspaceServer) validateRequest(header *pdpb.RequestHeader) error { + if s.IsClosed() { + return ErrNotStarted + } + if header.GetClusterId() != s.clusterID { + return status.Errorf(codes.FailedPrecondition, "mismatch cluster id, need %d but got %d", s.clusterID, header.GetClusterId()) + } + return nil +} + // LoadKeyspace load and return target keyspace metadata. // Request must specify keyspace name. // On Error, keyspaceMeta in response will be nil, // error information will be encoded in response header with corresponding error type. func (s *KeyspaceServer) LoadKeyspace(_ context.Context, request *keyspacepb.LoadKeyspaceRequest) (*keyspacepb.LoadKeyspaceResponse, error) { + if err := s.validateRequest(request.GetHeader()); err != nil { + return nil, err + } rc := s.GetRaftCluster() if rc == nil { return &keyspacepb.LoadKeyspaceResponse{Header: s.notBootstrappedHeader()}, nil @@ -93,7 +82,10 @@ func (s *KeyspaceServer) LoadKeyspace(_ context.Context, request *keyspacepb.Loa // WatchKeyspaces captures and sends keyspace metadata changes to the client via gRPC stream. // Note: It sends all existing keyspaces as it's first package to the client. -func (s *KeyspaceServer) WatchKeyspaces(_ *keyspacepb.WatchKeyspacesRequest, stream keyspacepb.Keyspace_WatchKeyspacesServer) error { +func (s *KeyspaceServer) WatchKeyspaces(request *keyspacepb.WatchKeyspacesRequest, stream keyspacepb.Keyspace_WatchKeyspacesServer) error { + if err := s.validateRequest(request.GetHeader()); err != nil { + return err + } rc := s.GetRaftCluster() if rc == nil { return stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.notBootstrappedHeader()}) diff --git a/server/server.go b/server/server.go index 49c80c05e33..1f9393e8dbb 100644 --- a/server/server.go +++ b/server/server.go @@ -282,8 +282,9 @@ func CreateServer(ctx context.Context, cfg *config.Config, serviceBuilders ...Ha etcdCfg.UserHandlers = userHandlers } etcdCfg.ServiceRegister = func(gs *grpc.Server) { - pdpb.RegisterPDServer(gs, &GrpcServer{Server: s}) - keyspacepb.RegisterKeyspaceServer(gs, &KeyspaceServer{Server: s}) + grpcServer := &GrpcServer{Server: s} + pdpb.RegisterPDServer(gs, grpcServer) + keyspacepb.RegisterKeyspaceServer(gs, &KeyspaceServer{GrpcServer: grpcServer}) diagnosticspb.RegisterDiagnosticsServer(gs, s) } s.etcdCfg = etcdCfg From 73601937fa3f74a33c2dc5082e784c2479ab272e Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 9 Aug 2022 15:00:22 +0800 Subject: [PATCH 55/76] make lockGroup more general Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- pkg/syncutil/lock_group.go | 55 +++++++++++++++++++-------------- pkg/syncutil/lock_group_test.go | 6 ++-- server/keyspace/keyspace.go | 2 +- server/keyspace/util.go | 9 ++++++ 4 files changed, 45 insertions(+), 27 deletions(-) diff --git a/pkg/syncutil/lock_group.go b/pkg/syncutil/lock_group.go index 1a92d164a8a..e6abe21c9e8 100644 --- a/pkg/syncutil/lock_group.go +++ b/pkg/syncutil/lock_group.go @@ -16,36 +16,45 @@ package syncutil import "fmt" -// LockGroup is a map of mutex that locks each keyspace separately. -// It's used to guarantee that operations on different keyspace won't block each other. +// LockGroup is a map of mutex that locks entries with different id separately. +// It's used levitate lock contentions of using a global lock. type LockGroup struct { groupLock Mutex // protects group. - entries map[uint32]*Mutex // map of locks with keyspaceID as key. - // hashFn hashes spaceID to map key, it's main purpose is to limit the total - // number of mutexes in the group, as using a mutex for every keyspace is too memory heavy. - hashFn func(spaceID uint32) uint32 + entries map[uint32]*Mutex // map of locks with id as key. + // hashFn hashes id to map key, it's main purpose is to limit the total + // number of mutexes in the group, as using a mutex for every id is too memory heavy. + hashFn func(id uint32) uint32 +} + +// LockGroupOption configures the lock group. +type LockGroupOption func(lg *LockGroup) + +// WithHash sets the lockGroup's hash function to provided hashFn. +func WithHash(hashFn func(id uint32) uint32) LockGroupOption { + return func(lg *LockGroup) { + lg.hashFn = hashFn + } } // NewLockGroup create and return an empty lockGroup. -func NewLockGroup() *LockGroup { - return &LockGroup{ +func NewLockGroup(options ...LockGroupOption) *LockGroup { + lockGroup := &LockGroup{ entries: make(map[uint32]*Mutex), - // A simple mask is applied to spaceID to use its last byte as map key, - // limiting the maximum map length to 256. - // Since keyspaceID is sequentially allocated, this can also reduce the chance - // of collision when comparing with random hashes. - hashFn: func(spaceID uint32) uint32 { - return spaceID & 0xFF - }, + // If no custom hash function provided, use identity hash. + hashFn: func(id uint32) uint32 { return id }, + } + for _, op := range options { + op(lockGroup) } + return lockGroup } -// Lock locks the given keyspace based on the hash of the spaceID. -func (g *LockGroup) Lock(spaceID uint32) { +// Lock locks the target mutex base on the hash of id. +func (g *LockGroup) Lock(id uint32) { g.groupLock.Lock() - hashedID := spaceID & 0xFF + hashedID := g.hashFn(id) e, ok := g.entries[hashedID] - // If target keyspace's lock has not been initialized, create a new lock. + // If target id's lock has not been initialized, create a new lock. if !ok { e = &Mutex{} g.entries[hashedID] = e @@ -54,15 +63,15 @@ func (g *LockGroup) Lock(spaceID uint32) { e.Lock() } -// Unlock unlocks the keyspace based on the hash of the spaceID. -func (g *LockGroup) Unlock(spaceID uint32) { +// Unlock unlocks the target mutex based on the hash of the id. +func (g *LockGroup) Unlock(id uint32) { g.groupLock.Lock() - hashedID := spaceID & 0xFF + hashedID := g.hashFn(id) e, ok := g.entries[hashedID] if !ok { // Entry must exist, otherwise there should be a run-time error and panic. g.groupLock.Unlock() - panic(fmt.Errorf("unlock requested for key %v, but no entry found", spaceID)) + panic(fmt.Errorf("unlock requested for key %v, but no entry found", id)) } g.groupLock.Unlock() e.Unlock() diff --git a/pkg/syncutil/lock_group_test.go b/pkg/syncutil/lock_group_test.go index a33c1deb1a6..458e75abaca 100644 --- a/pkg/syncutil/lock_group_test.go +++ b/pkg/syncutil/lock_group_test.go @@ -24,8 +24,8 @@ import ( func TestLockGroup(t *testing.T) { re := require.New(t) - group := NewLockGroup() - concurrency := 20 + group := NewLockGroup(WithHash(func(id uint32) uint32 { return id & 0xFF })) + concurrency := 300 var wg sync.WaitGroup wg.Add(concurrency) for i := 0; i < concurrency; i++ { @@ -41,7 +41,7 @@ func TestLockGroup(t *testing.T) { // mustSequentialUpdateSingle checks that for any given update, update is sequential. func mustSequentialUpdateSingle(re *require.Assertions, spaceID uint32, group *LockGroup) { - concurrency := 100 + concurrency := 50 total := 0 var wg sync.WaitGroup wg.Add(concurrency) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index 8e2e73406a8..8b3b66e9ce2 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -63,7 +63,7 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator manager := &Manager{ store: store, idAllocator: idAllocator, - metaLock: syncutil.NewLockGroup(), + metaLock: syncutil.NewLockGroup(syncutil.WithHash(SpaceIDHash)), } now := time.Now() // Initialize default keyspace. diff --git a/server/keyspace/util.go b/server/keyspace/util.go index 5c6072a84fb..666052e6dad 100644 --- a/server/keyspace/util.go +++ b/server/keyspace/util.go @@ -68,3 +68,12 @@ func validateName(name string) error { } return nil } + +// SpaceIDHash is used to hash the spaceID inside the lockGroup. +// A simple mask is applied to spaceID to use its last byte as map key, +// limiting the maximum map length to 256. +// Since keyspaceID is sequentially allocated, this can also reduce the chance +// of collision when comparing with random hashes. +func SpaceIDHash(spaceID uint32) uint32 { + return spaceID & 0xFF +} From 517b990165fb1c70e5c259c83ea0291c81539a5c Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 10 Aug 2022 11:28:18 +0800 Subject: [PATCH 56/76] added init keyspace check Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index 8b3b66e9ce2..7551e6efb07 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -65,8 +65,16 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator idAllocator: idAllocator, metaLock: syncutil.NewLockGroup(syncutil.WithHash(SpaceIDHash)), } - now := time.Now() + // If default keyspace already exists, skip initialization. + defaultExist, _, err := manager.store.LoadKeyspaceIDByName(DefaultKeyspaceName) + if err != nil { + return nil, err + } + if defaultExist { + return manager, nil + } // Initialize default keyspace. + now := time.Now() defaultKeyspace := &keyspacepb.KeyspaceMeta{ Id: DefaultKeyspaceID, Name: DefaultKeyspaceName, @@ -74,8 +82,8 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator CreatedAt: now.Unix(), StateChangedAt: now.Unix(), } - _, err := manager.saveNewKeyspace(defaultKeyspace) - if err != nil { + _, err = manager.saveNewKeyspace(defaultKeyspace) + if err != nil && err != ErrKeyspaceExists { return nil, err } return manager, nil From 23cdc0a1f52302962924a062cd1f86c981425ba2 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 10 Aug 2022 11:28:36 +0800 Subject: [PATCH 57/76] fixed dial client leak Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- tests/server/apiv2/handlers/keyspace_test.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index 3278de9fc62..a0cd36a6e40 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -19,6 +19,10 @@ import ( "context" "encoding/json" "fmt" + "io" + "net/http" + "testing" + "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -27,9 +31,6 @@ import ( "github.com/tikv/pd/server/keyspace" "github.com/tikv/pd/tests" "go.uber.org/goleak" - "io" - "net/http" - "testing" ) const keyspacesPrefix = "/pd/api/v2/keyspaces" @@ -153,12 +154,12 @@ func sendLoadRangeRequest(re *require.Assertions, server *tests.TestServer, toke httpReq.URL.RawQuery = query.Encode() // Send request. httpResp, err := dialClient.Do(httpReq) - re.Equal(http.StatusOK, httpResp.StatusCode) re.NoError(err) + defer httpResp.Body.Close() + re.Equal(http.StatusOK, httpResp.StatusCode) // Receive & decode response. data, err := io.ReadAll(httpResp.Body) re.NoError(err) - re.NoError(httpResp.Body.Close()) resp := &handlers.LoadAllKeyspacesResponse{} re.NoError(json.Unmarshal(data, resp)) return resp @@ -169,12 +170,12 @@ func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, na re.NoError(err) resp, err := dialClient.Do(httpReq) re.NoError(err) + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return false, nil } data, err := io.ReadAll(resp.Body) re.NoError(err) - re.NoError(resp.Body.Close()) meta := &handlers.KeyspaceMeta{} re.NoError(json.Unmarshal(data, meta)) return true, meta.KeyspaceMeta @@ -202,10 +203,10 @@ func mustCreateKeyspace(re *require.Assertions, server *tests.TestServer, reques re.NoError(err) resp, err := dialClient.Do(httpReq) re.NoError(err) + defer resp.Body.Close() re.Equal(http.StatusOK, resp.StatusCode) data, err = io.ReadAll(resp.Body) re.NoError(err) - re.NoError(resp.Body.Close()) meta := &handlers.KeyspaceMeta{} re.NoError(json.Unmarshal(data, meta)) checkCreateRequest(re, request, meta.KeyspaceMeta) @@ -219,10 +220,10 @@ func mustUpdateKeyspaceConfig(re *require.Assertions, server *tests.TestServer, re.NoError(err) resp, err := dialClient.Do(httpReq) re.NoError(err) + defer resp.Body.Close() re.Equal(http.StatusOK, resp.StatusCode) data, err = io.ReadAll(resp.Body) re.NoError(err) - re.NoError(resp.Body.Close()) meta := &handlers.KeyspaceMeta{} re.NoError(json.Unmarshal(data, meta)) return meta.KeyspaceMeta @@ -231,10 +232,10 @@ func mustUpdateKeyspaceConfig(re *require.Assertions, server *tests.TestServer, func mustLoadKeyspaces(re *require.Assertions, server *tests.TestServer, name string) *keyspacepb.KeyspaceMeta { resp, err := dialClient.Get(server.GetAddr() + keyspacesPrefix + "/" + name) re.NoError(err) + defer resp.Body.Close() re.Equal(http.StatusOK, resp.StatusCode) data, err := io.ReadAll(resp.Body) re.NoError(err) - re.NoError(resp.Body.Close()) meta := &handlers.KeyspaceMeta{} re.NoError(json.Unmarshal(data, meta)) return meta.KeyspaceMeta From 9b274df39efb5034bc79915d3fb0fc7bd87965de Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 10 Aug 2022 11:35:39 +0800 Subject: [PATCH 58/76] lower lockgroup test concurrency Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- pkg/syncutil/lock_group_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/syncutil/lock_group_test.go b/pkg/syncutil/lock_group_test.go index 458e75abaca..4e7dd123700 100644 --- a/pkg/syncutil/lock_group_test.go +++ b/pkg/syncutil/lock_group_test.go @@ -24,8 +24,8 @@ import ( func TestLockGroup(t *testing.T) { re := require.New(t) - group := NewLockGroup(WithHash(func(id uint32) uint32 { return id & 0xFF })) - concurrency := 300 + group := NewLockGroup(WithHash(func(id uint32) uint32 { return id & 0xF })) + concurrency := 50 var wg sync.WaitGroup wg.Add(concurrency) for i := 0; i < concurrency; i++ { @@ -36,7 +36,7 @@ func TestLockGroup(t *testing.T) { } wg.Wait() // Check that size of the lock group is limited. - re.LessOrEqual(len(group.entries), 1<<8) + re.LessOrEqual(len(group.entries), 16) } // mustSequentialUpdateSingle checks that for any given update, update is sequential. From e91776c64164a6613767f6a4d9551d01c5a93ef1 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 10 Aug 2022 13:14:30 +0800 Subject: [PATCH 59/76] cleanup cluster before watch keyspace test Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- tests/client/keyspace_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/client/keyspace_test.go b/tests/client/keyspace_test.go index 035d944f540..b63bb7edaea 100644 --- a/tests/client/keyspace_test.go +++ b/tests/client/keyspace_test.go @@ -16,11 +16,12 @@ package client_test import ( "fmt" + "time" + "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" "github.com/tikv/pd/server" "github.com/tikv/pd/server/keyspace" - "time" ) const ( @@ -49,7 +50,7 @@ func mustMakeTestKeyspaces(re *require.Assertions, server *server.Server, start, func (suite *clientTestSuite) TestLoadKeyspace() { re := suite.Require() - metas := mustMakeTestKeyspaces(re, suite.srv, 0, 10) + metas := mustMakeTestKeyspaces(re, suite.srv, 100, 10) for _, expected := range metas { loaded, err := suite.client.LoadKeyspace(suite.ctx, expected.Name) re.NoError(err) @@ -65,7 +66,10 @@ func (suite *clientTestSuite) TestLoadKeyspace() { re.Equal(keyspace.DefaultKeyspaceName, keyspaceDefault.Name) } -func (suite *clientTestSuite) TestWatchKeyspace() { +func (suite *clientTestSuite) TestWatchKeyspaces() { + // Use a new cluster test watch keyspace. + suite.TearDownSuite() + suite.SetupSuite() re := suite.Require() initialKeyspaces := mustMakeTestKeyspaces(re, suite.srv, 0, 10) watchChan, err := suite.client.WatchKeyspaces(suite.ctx) From bbac817115562dc5cde973f86cb32c6360d9c825 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 10 Aug 2022 14:18:21 +0800 Subject: [PATCH 60/76] typo fix Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace_service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 9ae280abba6..408c374e5ff 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -72,7 +72,7 @@ func (s *KeyspaceServer) LoadKeyspace(_ context.Context, request *keyspacepb.Loa manager := s.GetKeyspaceManager() meta, err := manager.LoadKeyspace(request.Name) if err != nil { - return &keyspacepb.LoadKeyspaceResponse{Header: s.getErrorHeader(err)}, err + return &keyspacepb.LoadKeyspaceResponse{Header: s.getErrorHeader(err)}, nil } return &keyspacepb.LoadKeyspaceResponse{ Header: s.header(), From bd6b9acb484e66e0ac2e690814a5e104f5eb11ae Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 10 Aug 2022 15:11:29 +0800 Subject: [PATCH 61/76] reinstate checks for leadership Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace_service.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 408c374e5ff..40c3d8145c0 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -24,8 +24,6 @@ import ( "github.com/tikv/pd/server/keyspace" "github.com/tikv/pd/server/storage/endpoint" "go.etcd.io/etcd/clientv3" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) // KeyspaceServer wraps GrpcServer to provide keyspace service. @@ -45,17 +43,6 @@ func (s *KeyspaceServer) getErrorHeader(err error) *pdpb.ResponseHeader { } } -// validateRequest checks that server is serving and cluster id matches the requested. -func (s *KeyspaceServer) validateRequest(header *pdpb.RequestHeader) error { - if s.IsClosed() { - return ErrNotStarted - } - if header.GetClusterId() != s.clusterID { - return status.Errorf(codes.FailedPrecondition, "mismatch cluster id, need %d but got %d", s.clusterID, header.GetClusterId()) - } - return nil -} - // LoadKeyspace load and return target keyspace metadata. // Request must specify keyspace name. // On Error, keyspaceMeta in response will be nil, From 9dda2920772bb6ab4f56e48f9dc97bfd83d20ba5 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 10 Aug 2022 15:59:18 +0800 Subject: [PATCH 62/76] reformat import for client Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- client/keyspace_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/keyspace_client.go b/client/keyspace_client.go index a469a45b00e..f8a0dd04bb3 100644 --- a/client/keyspace_client.go +++ b/client/keyspace_client.go @@ -16,7 +16,6 @@ package pd import ( "context" - "go.uber.org/zap" "time" "github.com/opentracing/opentracing-go" @@ -24,6 +23,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/log" "github.com/tikv/pd/client/grpcutil" + "go.uber.org/zap" "google.golang.org/grpc" ) From af3d314f3c39fa333fe08f3a973162b7d1d6458f Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 31 Aug 2022 16:47:54 +0800 Subject: [PATCH 63/76] update keyspace api to new manager requests Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index 6c561f79c95..72ff0893f05 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -68,7 +68,7 @@ func CreateKeyspace(c *gin.Context) { req := &keyspace.CreateKeyspaceRequest{ Name: createParams.Name, Config: createParams.Config, - Now: time.Now(), + Now: time.Now().Unix(), } meta, err := manager.CreateKeyspace(req) if err != nil { @@ -289,7 +289,7 @@ func updateKeyspaceState(c *gin.Context, state keyspacepb.KeyspaceState) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() name := c.Param("name") - meta, err := manager.UpdateKeyspaceState(name, state, time.Now()) + meta, err := manager.UpdateKeyspaceState(name, state, time.Now().Unix()) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) return From a31704e8bea892b34c78c261383aed2b427087d2 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 1 Sep 2022 20:01:19 +0800 Subject: [PATCH 64/76] update keyspace api to RESTful Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 67 +++++++++----------- tests/server/apiv2/handlers/keyspace_test.go | 23 ++++--- 2 files changed, 43 insertions(+), 47 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index 72ff0893f05..fd50ded5d16 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -21,6 +21,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/server" @@ -35,10 +36,8 @@ func RegisterKeyspace(r *gin.RouterGroup) { router.POST("", CreateKeyspace) router.GET("", LoadAllKeyspaces) router.GET("/:name", LoadKeyspace) - router.POST("/:name/update-config", UpdateKeyspaceConfig) - router.POST("/:name/enable", EnableKeyspace) - router.POST("/:name/disable", DisableKeyspace) - router.POST("/:name/archive", ArchiveKeyspace) + router.PATCH("/:name/config", UpdateKeyspaceConfig) + router.PUT("/:name/state", UpdateKeyspaceState) } // CreateKeyspaceParams represents parameters needed when creating a new keyspace. @@ -54,6 +53,7 @@ type CreateKeyspaceParams struct { // @Param body body CreateKeyspaceParams true "Create keyspace parameters" // @Produce json // @Success 200 {object} KeyspaceMeta +// @Failure 400 {string} string "The input is invalid." // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /keyspaces [post] func CreateKeyspace(c *gin.Context) { @@ -151,6 +151,7 @@ type LoadAllKeyspacesResponse struct { // @Param limit query string false "maximum number of results to return" // @Produce json // @Success 200 {object} LoadAllKeyspacesResponse +// @Failure 400 {string} string "The input is invalid." // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /keyspaces [get] func LoadAllKeyspaces(c *gin.Context) { @@ -201,7 +202,7 @@ type UpdateConfigParams struct { } // UpdateKeyspaceConfig updates target keyspace's config. -// This is a custom action that supports PATCH semantics and uses the JSON merge patch +// This api uses PATCH semantic and supports JSON Merge Patch. // format and processing rules. // @Tags keyspaces // @Summary Update keyspace config. @@ -209,8 +210,9 @@ type UpdateConfigParams struct { // @Param body body UpdateConfigParams true "Update keyspace parameters" // @Produce json // @Success 200 {object} KeyspaceMeta +// @Failure 400 {string} string "The input is invalid." // @Failure 500 {string} string "PD server failed to proceed the request." -// Router /keyspaces/{name}/update-config [post] +// Router /keyspaces/{name}/update-config [patch] func UpdateKeyspaceConfig(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() @@ -250,46 +252,37 @@ func getMutations(patch map[string]*string) []*keyspace.Mutation { return mutations } -// EnableKeyspace enables target keyspace. -// @Tags keyspaces -// @Summary Enable keyspace. -// @Param name path string true "Keyspace Name" -// @Produce json -// @Success 200 {object} KeyspaceMeta -// @Failure 500 {string} string "PD server failed to proceed the request." -// Router /keyspaces/{name}/enable [post] -func EnableKeyspace(c *gin.Context) { - updateKeyspaceState(c, keyspacepb.KeyspaceState_ENABLED) -} - -// DisableKeyspace disables target keyspace. -// @Tags keyspaces -// @Summary Disable keyspace. -// @Param name path string true "Keyspace Name" -// @Produce json -// @Success 200 {object} KeyspaceMeta -// @Failure 500 {string} string "PD server failed to proceed the request." -// Router /keyspaces/{name}/disable [post] -func DisableKeyspace(c *gin.Context) { - updateKeyspaceState(c, keyspacepb.KeyspaceState_DISABLED) +// UpdateStateParam represents parameters needed to modify target keyspace's state. +// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +type UpdateStateParam struct { + State string `json:"state"` } -// ArchiveKeyspace archives target keyspace. +// UpdateKeyspaceState update the target keyspace's state. // @Tags keyspaces -// @Summary Archive keyspace. +// @Summary Update keyspace state. // @Param name path string true "Keyspace Name" +// @Param body body UpdateStateParam true "New state for the keyspace" // @Produce json // @Success 200 {object} KeyspaceMeta +// @Failure 400 {string} string "The input is invalid." // @Failure 500 {string} string "PD server failed to proceed the request." -// Router /keyspaces/{name}/archive [post] -func ArchiveKeyspace(c *gin.Context) { - updateKeyspaceState(c, keyspacepb.KeyspaceState_ARCHIVED) -} -func updateKeyspaceState(c *gin.Context, state keyspacepb.KeyspaceState) { +// Router /keyspaces/{name}/archive [put] +func UpdateKeyspaceState(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() name := c.Param("name") - meta, err := manager.UpdateKeyspaceState(name, state, time.Now().Unix()) + param := &UpdateStateParam{} + err := c.BindJSON(param) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) + return + } + targetState, ok := keyspacepb.KeyspaceState_value[param.State] + if !ok { + c.AbortWithStatusJSON(http.StatusBadRequest, errors.Errorf("unknown target state: %s", param.State)) + } + meta, err := manager.UpdateKeyspaceState(name, keyspacepb.KeyspaceState(targetState), time.Now().Unix()) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) return @@ -314,7 +307,7 @@ func (meta *KeyspaceMeta) MarshalJSON() ([]byte, error) { Config map[string]string `json:"config,omitempty"` }{ meta.Name, - keyspacepb.KeyspaceState_name[int32(meta.State)], + meta.State.String(), meta.CreatedAt, meta.StateChangedAt, meta.Config, diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index a0cd36a6e40..3fc517dcb63 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -107,26 +107,26 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { keyspaces := mustMakeTestKeyspaces(re, suite.server, 10) for _, created := range keyspaces { // Should NOT allow archiving ENABLED keyspace. - success, _ := sendUpdateStateRequest(re, suite.server, created.Name, "archive") + success, _ := sendUpdateStateRequest(re, suite.server, created.Name, "ARCHIVED") re.False(success) // Disabling an ENABLED keyspace is allowed. - success, disabled := sendUpdateStateRequest(re, suite.server, created.Name, "disable") + success, disabled := sendUpdateStateRequest(re, suite.server, created.Name, "DISABLED") re.True(success) re.Equal(keyspacepb.KeyspaceState_DISABLED, disabled.State) // Disabling an already DISABLED keyspace should not result in any change. - success, disabledAgain := sendUpdateStateRequest(re, suite.server, created.Name, "disable") + success, disabledAgain := sendUpdateStateRequest(re, suite.server, created.Name, "DISABLED") re.True(success) re.Equal(disabled, disabledAgain) // Archiving a DISABLED keyspace should be allowed. - success, archived := sendUpdateStateRequest(re, suite.server, created.Name, "archive") + success, archived := sendUpdateStateRequest(re, suite.server, created.Name, "ARCHIVED") re.True(success) re.Equal(keyspacepb.KeyspaceState_ARCHIVED, archived.State) // Modifying ARCHIVED keyspace is not allowed. - success, _ = sendUpdateStateRequest(re, suite.server, created.Name, "disable") + success, _ = sendUpdateStateRequest(re, suite.server, created.Name, "DISABLED") re.False(success) } // Changing default keyspace's state is NOT allowed. - success, _ := sendUpdateStateRequest(re, suite.server, keyspace.DefaultKeyspaceName, "disable") + success, _ := sendUpdateStateRequest(re, suite.server, keyspace.DefaultKeyspaceName, "DISABLED") re.False(success) } @@ -165,8 +165,11 @@ func sendLoadRangeRequest(re *require.Assertions, server *tests.TestServer, toke return resp } -func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, name, action string) (bool, *keyspacepb.KeyspaceMeta) { - httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspacesPrefix+"/"+name+"/"+action, nil) +func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, name, newState string) (bool, *keyspacepb.KeyspaceMeta) { + request := handlers.UpdateStateParam{State: newState} + data, err := json.Marshal(request) + re.NoError(err) + httpReq, err := http.NewRequest(http.MethodPut, server.GetAddr()+keyspacesPrefix+"/"+name+"/state", bytes.NewBuffer(data)) re.NoError(err) resp, err := dialClient.Do(httpReq) re.NoError(err) @@ -174,7 +177,7 @@ func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, na if resp.StatusCode != http.StatusOK { return false, nil } - data, err := io.ReadAll(resp.Body) + data, err = io.ReadAll(resp.Body) re.NoError(err) meta := &handlers.KeyspaceMeta{} re.NoError(json.Unmarshal(data, meta)) @@ -216,7 +219,7 @@ func mustCreateKeyspace(re *require.Assertions, server *tests.TestServer, reques func mustUpdateKeyspaceConfig(re *require.Assertions, server *tests.TestServer, name string, request *handlers.UpdateConfigParams) *keyspacepb.KeyspaceMeta { data, err := json.Marshal(request) re.NoError(err) - httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspacesPrefix+"/"+name+"/update-config", bytes.NewBuffer(data)) + httpReq, err := http.NewRequest(http.MethodPatch, server.GetAddr()+keyspacesPrefix+"/"+name+"/config", bytes.NewBuffer(data)) re.NoError(err) resp, err := dialClient.Do(httpReq) re.NoError(err) From 2dca250aca8cb819cb84419035bd5e3b0ec6fcd4 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 1 Sep 2022 20:28:42 +0800 Subject: [PATCH 65/76] swag fmt Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 80 +++++++++++++++---------------- server/apiv2/router.go | 9 ++++ 2 files changed, 49 insertions(+), 40 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index fd50ded5d16..6e8a908dc34 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -48,14 +48,14 @@ type CreateKeyspaceParams struct { } // CreateKeyspace creates keyspace according to given input. -// @Tags keyspaces -// @Summary Create new keyspace. -// @Param body body CreateKeyspaceParams true "Create keyspace parameters" -// @Produce json -// @Success 200 {object} KeyspaceMeta -// @Failure 400 {string} string "The input is invalid." -// @Failure 500 {string} string "PD server failed to proceed the request." -// @Router /keyspaces [post] +// @Tags keyspaces +// @Summary Create new keyspace. +// @Param body body CreateKeyspaceParams true "Create keyspace parameters" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /keyspaces [post] func CreateKeyspace(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() @@ -79,13 +79,13 @@ func CreateKeyspace(c *gin.Context) { } // LoadKeyspace returns target keyspace. -// @Tags keyspaces -// @Summary Get keyspace info. -// @Param name path string true "Keyspace Name" -// @Produce json -// @Success 200 {object} KeyspaceMeta -// @Failure 500 {string} string "PD server failed to proceed the request." -// @Router /keyspaces/{name} [get] +// @Tags keyspaces +// @Summary Get keyspace info. +// @Param name path string true "Keyspace Name" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /keyspaces/{name} [get] func LoadKeyspace(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() @@ -145,15 +145,15 @@ type LoadAllKeyspacesResponse struct { } // LoadAllKeyspaces loads range of keyspaces. -// @Tags keyspaces -// @Summary list keyspaces. -// @Param page_token query string false "page token" -// @Param limit query string false "maximum number of results to return" -// @Produce json -// @Success 200 {object} LoadAllKeyspacesResponse -// @Failure 400 {string} string "The input is invalid." -// @Failure 500 {string} string "PD server failed to proceed the request." -// @Router /keyspaces [get] +// @Tags keyspaces +// @Summary list keyspaces. +// @Param page_token query string false "page token" +// @Param limit query string false "maximum number of results to return" +// @Produce json +// @Success 200 {object} LoadAllKeyspacesResponse +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /keyspaces [get] func LoadAllKeyspaces(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() @@ -204,14 +204,14 @@ type UpdateConfigParams struct { // UpdateKeyspaceConfig updates target keyspace's config. // This api uses PATCH semantic and supports JSON Merge Patch. // format and processing rules. -// @Tags keyspaces -// @Summary Update keyspace config. -// @Param name path string true "Keyspace Name" -// @Param body body UpdateConfigParams true "Update keyspace parameters" -// @Produce json -// @Success 200 {object} KeyspaceMeta -// @Failure 400 {string} string "The input is invalid." -// @Failure 500 {string} string "PD server failed to proceed the request." +// @Tags keyspaces +// @Summary Update keyspace config. +// @Param name path string true "Keyspace Name" +// @Param body body UpdateConfigParams true "Update keyspace parameters" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." // Router /keyspaces/{name}/update-config [patch] func UpdateKeyspaceConfig(c *gin.Context) { svr := c.MustGet("server").(*server.Server) @@ -259,14 +259,14 @@ type UpdateStateParam struct { } // UpdateKeyspaceState update the target keyspace's state. -// @Tags keyspaces -// @Summary Update keyspace state. -// @Param name path string true "Keyspace Name" -// @Param body body UpdateStateParam true "New state for the keyspace" -// @Produce json -// @Success 200 {object} KeyspaceMeta -// @Failure 400 {string} string "The input is invalid." -// @Failure 500 {string} string "PD server failed to proceed the request." +// @Tags keyspaces +// @Summary Update keyspace state. +// @Param name path string true "Keyspace Name" +// @Param body body UpdateStateParam true "New state for the keyspace" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." // Router /keyspaces/{name}/archive [put] func UpdateKeyspaceState(c *gin.Context) { svr := c.MustGet("server").(*server.Server) diff --git a/server/apiv2/router.go b/server/apiv2/router.go index 9f106fa22cb..f7aac8d1969 100644 --- a/server/apiv2/router.go +++ b/server/apiv2/router.go @@ -38,6 +38,15 @@ var group = server.ServiceGroup{ const apiV2Prefix = "/pd/api/v2/" // NewV2Handler creates a HTTP handler for API. +// @title Placement Driver Core API +// @version 2.0 +// @description This is placement driver. +// @contact.name Placement Driver Support +// @contact.url https://github.com/tikv/pd/issues +// @contact.email info@pingcap.com +// @license.name Apache 2.0 +// @license.url http://www.apache.org/licenses/LICENSE-2.0.html +// @BasePath /pd/api/v2 func NewV2Handler(_ context.Context, svr *server.Server) (http.Handler, server.ServiceGroup, error) { once.Do(func() { // See https://github.com/pingcap/tidb-dashboard/blob/f8ecb64e3d63f4ed91c3dca7a04362418ade01d8/pkg/apiserver/apiserver.go#L84 From 05cf81c84e346e6739c538cbb76c11bf4c2561d2 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 2 Sep 2022 11:54:23 +0800 Subject: [PATCH 66/76] address comments Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 5 +++-- tests/server/apiv2/handlers/keyspace_test.go | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index 6e8a908dc34..5e7321faf73 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -212,7 +212,7 @@ type UpdateConfigParams struct { // @Success 200 {object} KeyspaceMeta // @Failure 400 {string} string "The input is invalid." // @Failure 500 {string} string "PD server failed to proceed the request." -// Router /keyspaces/{name}/update-config [patch] +// Router /keyspaces/{name}/config [patch] func UpdateKeyspaceConfig(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() @@ -267,7 +267,7 @@ type UpdateStateParam struct { // @Success 200 {object} KeyspaceMeta // @Failure 400 {string} string "The input is invalid." // @Failure 500 {string} string "PD server failed to proceed the request." -// Router /keyspaces/{name}/archive [put] +// Router /keyspaces/{name}/state [put] func UpdateKeyspaceState(c *gin.Context) { svr := c.MustGet("server").(*server.Server) manager := svr.GetKeyspaceManager() @@ -281,6 +281,7 @@ func UpdateKeyspaceState(c *gin.Context) { targetState, ok := keyspacepb.KeyspaceState_value[param.State] if !ok { c.AbortWithStatusJSON(http.StatusBadRequest, errors.Errorf("unknown target state: %s", param.State)) + return } meta, err := manager.UpdateKeyspaceState(name, keyspacepb.KeyspaceState(targetState), time.Now().Unix()) if err != nil { diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index 3fc517dcb63..560215a01bc 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -60,7 +60,7 @@ func TestKeyspaceTestSuite(t *testing.T) { func (suite *keyspaceTestSuite) SetupTest() { ctx, cancel := context.WithCancel(context.Background()) suite.cleanup = cancel - cluster, err := tests.NewTestCluster(ctx, 3) + cluster, err := tests.NewTestCluster(ctx, 1) suite.cluster = cluster suite.NoError(err) suite.NoError(cluster.RunInitialServers()) @@ -124,6 +124,9 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { // Modifying ARCHIVED keyspace is not allowed. success, _ = sendUpdateStateRequest(re, suite.server, created.Name, "DISABLED") re.False(success) + // Using illegal state is not allowed. + success, _ = sendUpdateStateRequest(re, suite.server, created.Name, "UNKNOWN") + re.False(success) } // Changing default keyspace's state is NOT allowed. success, _ := sendUpdateStateRequest(re, suite.server, keyspace.DefaultKeyspaceName, "DISABLED") From be810393262adac66e093b8e998b06f8dc467f21 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 4 Jan 2023 15:47:39 +0800 Subject: [PATCH 67/76] sort import Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- pkg/storage/kv/etcd_kv.go | 3 ++- pkg/storage/kv/levedb_kv.go | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/storage/kv/etcd_kv.go b/pkg/storage/kv/etcd_kv.go index 4348585233e..cce3e5200d9 100644 --- a/pkg/storage/kv/etcd_kv.go +++ b/pkg/storage/kv/etcd_kv.go @@ -23,8 +23,9 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/log" - "github.com/tikv/pd/pkg/utils/syncutil" + "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/syncutil" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" ) diff --git a/pkg/storage/kv/levedb_kv.go b/pkg/storage/kv/levedb_kv.go index 248af585e47..6f93cd0237f 100644 --- a/pkg/storage/kv/levedb_kv.go +++ b/pkg/storage/kv/levedb_kv.go @@ -17,12 +17,11 @@ package kv import ( "context" - "github.com/gogo/protobuf/proto" "github.com/pingcap/errors" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/syncutil" + "github.com/tikv/pd/pkg/utils/syncutil" ) // LevelDBKV is a kv store using LevelDB. From 149ce0c732e9f1a4ccccbcae1bb305516999e58f Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 9 Jan 2023 17:18:46 +0800 Subject: [PATCH 68/76] pin create revision when key not exist Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- pkg/storage/kv/etcd_kv.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/pkg/storage/kv/etcd_kv.go b/pkg/storage/kv/etcd_kv.go index cce3e5200d9..fe33d868070 100644 --- a/pkg/storage/kv/etcd_kv.go +++ b/pkg/storage/kv/etcd_kv.go @@ -223,18 +223,32 @@ func (txn *etcdTxn) Remove(key string) error { // Load loads the target value from etcd and puts a comparator into conditions. func (txn *etcdTxn) Load(key string) (string, error) { - value, err := txn.kv.Load(key) - // If Load failed, preserve the failure behavior of base Load. + key = path.Join(txn.kv.rootPath, key) + resp, err := etcdutil.EtcdKVGet(txn.kv.client, key) if err != nil { - return value, err + return "", err + } + var condition clientv3.Cmp + var value string + switch respLen := len(resp.Kvs); { + case respLen == 0: + // If target key does not contain a value, pin the CreateRevision of the key to 0. + // Returned value should be empty string. + value = "" + condition = clientv3.Compare(clientv3.CreateRevision(key), "=", 0) + case respLen == 1: + // If target key has value, must make sure it stays the same at the time of commit. + value = string(resp.Kvs[0].Value) + condition = clientv3.Compare(clientv3.Value(key), "=", value) + default: + // If response contains multiple kvs, error occurred. + return "", errs.ErrEtcdKVGetResponse.GenWithStackByArgs(resp.Kvs) } - // If load successful, must make sure value stays the same before commit. - fullKey := path.Join(txn.kv.rootPath, key) - condition := clientv3.Compare(clientv3.Value(fullKey), "=", value) + // Append the check condition to transaction. txn.mu.Lock() defer txn.mu.Unlock() txn.conditions = append(txn.conditions, condition) - return value, err + return value, nil } // LoadRange loads the target range from etcd, From d5f462eee2d37c396161384924eafcc0d2637ade Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 9 Jan 2023 18:08:37 +0800 Subject: [PATCH 69/76] init commit Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- client/go.mod | 3 + client/go.sum | 4 +- client/keyspace_client.go | 44 +++- client/metrics.go | 2 + go.mod | 3 + go.sum | 9 +- pkg/storage/endpoint/keyspace.go | 108 +++++---- server/keyspace/keyspace.go | 392 ++++++++++++++++++++----------- server/keyspace/keyspace_test.go | 51 ++-- server/keyspace/util.go | 30 ++- server/keyspace_service.go | 23 +- 11 files changed, 436 insertions(+), 233 deletions(-) diff --git a/client/go.mod b/client/go.mod index 09277d77a4d..97cdab0e858 100644 --- a/client/go.mod +++ b/client/go.mod @@ -14,3 +14,6 @@ require ( go.uber.org/zap v1.20.0 google.golang.org/grpc v1.43.0 ) + +// TODO: remove after kvproto merge +replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349 diff --git a/client/go.sum b/client/go.sum index 32758cfe55e..63e05ac096a 100644 --- a/client/go.sum +++ b/client/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349 h1:aq9/uAnlenI/ensOMpOSvowqzEPQhIoZ9nmadMBCKNs= +github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -104,8 +106,6 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 h1:FYgKV9znRQmzVrrJDZ0gUfMIvKLAMU1tu1UKJib8bEQ= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/client/keyspace_client.go b/client/keyspace_client.go index a469a45b00e..f87bf6a9fac 100644 --- a/client/keyspace_client.go +++ b/client/keyspace_client.go @@ -16,7 +16,6 @@ package pd import ( "context" - "go.uber.org/zap" "time" "github.com/opentracing/opentracing-go" @@ -24,6 +23,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/log" "github.com/tikv/pd/client/grpcutil" + "go.uber.org/zap" "google.golang.org/grpc" ) @@ -33,6 +33,7 @@ type KeyspaceClient interface { LoadKeyspace(ctx context.Context, name string) (*keyspacepb.KeyspaceMeta, error) // WatchKeyspaces watches keyspace meta changes. WatchKeyspaces(ctx context.Context) (chan []*keyspacepb.KeyspaceMeta, error) + UpdateKeyspaceState(ctx context.Context, id uint32, state keyspacepb.KeyspaceState) (*keyspacepb.KeyspaceMeta, error) } // keyspaceClient returns the KeyspaceClient from current PD leader. @@ -111,3 +112,44 @@ func (c *client) WatchKeyspaces(ctx context.Context) (chan []*keyspacepb.Keyspac }() return keyspaceWatcherChan, err } + +// UpdateKeyspaceState attempts to update the keyspace specified by ID to the target state, +// it will also record StateChangedAt for the given keyspace if a state change took place. +// Currently, legal operations includes: +// +// ENABLED -> {ENABLED, DISABLED} +// DISABLED -> {ENABLED, DISABLED, ARCHIVED} +// ARCHIVED -> {ARCHIVED, TOMBSTONE} +// TOMBSTONE -> {TOMBSTONE} +// +// Updated keyspace meta will be returned. +func (c *client) UpdateKeyspaceState(ctx context.Context, id uint32, state keyspacepb.KeyspaceState) (*keyspacepb.KeyspaceMeta, error) { + if span := opentracing.SpanFromContext(ctx); span != nil { + span = opentracing.StartSpan("keyspaceClient.UpdateKeyspaceState", opentracing.ChildOf(span.Context())) + defer span.Finish() + } + start := time.Now() + defer func() { cmdDurationUpdateKeyspaceState.Observe(time.Since(start).Seconds()) }() + ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + req := &keyspacepb.UpdateKeyspaceStateRequest{ + Header: c.requestHeader(), + Id: id, + State: state, + } + ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) + resp, err := c.keyspaceClient().UpdateKeyspaceState(ctx, req) + cancel() + + if err != nil { + cmdFailedDurationUpdateKeyspaceState.Observe(time.Since(start).Seconds()) + c.ScheduleCheckLeader() + return nil, err + } + + if resp.Header.GetError() != nil { + cmdFailedDurationUpdateKeyspaceState.Observe(time.Since(start).Seconds()) + return nil, errors.Errorf("Update state for keyspace id %d failed: %s", id, resp.Header.GetError().String()) + } + + return resp.Keyspace, nil +} diff --git a/client/metrics.go b/client/metrics.go index b92b66727d6..9d713d78667 100644 --- a/client/metrics.go +++ b/client/metrics.go @@ -100,6 +100,7 @@ var ( cmdDurationSplitRegions = cmdDuration.WithLabelValues("split_regions") cmdDurationSplitAndScatterRegions = cmdDuration.WithLabelValues("split_and_scatter_regions") cmdDurationLoadKeyspace = cmdDuration.WithLabelValues("load_keyspace") + cmdDurationUpdateKeyspaceState = cmdDuration.WithLabelValues("update_keyspace_state") cmdFailDurationGetRegion = cmdFailedDuration.WithLabelValues("get_region") cmdFailDurationTSO = cmdFailedDuration.WithLabelValues("tso") @@ -112,6 +113,7 @@ var ( cmdFailedDurationUpdateGCSafePoint = cmdFailedDuration.WithLabelValues("update_gc_safe_point") cmdFailedDurationUpdateServiceGCSafePoint = cmdFailedDuration.WithLabelValues("update_service_gc_safe_point") cmdFailedDurationLoadKeyspace = cmdDuration.WithLabelValues("load_keyspace") + cmdFailedDurationUpdateKeyspaceState = cmdDuration.WithLabelValues("update_keyspace_state") requestDurationTSO = requestDuration.WithLabelValues("tso") ) diff --git a/go.mod b/go.mod index 23cf2ecbf39..002c09e43ec 100644 --- a/go.mod +++ b/go.mod @@ -189,3 +189,6 @@ require ( // kvproto at the same time. You can run `go mod tidy` to make it replaced with go-mod style specification. // After the PR to kvproto is merged, remember to comment this out and run `go mod tidy`. // replace github.com/pingcap/kvproto => github.com/$YourPrivateRepo $YourPrivateBranch + +// TODO: remove after kvproto merge +replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349 diff --git a/go.sum b/go.sum index 19c5daac87f..71070db31ff 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AlekSi/gocov-xml v1.0.0 h1:4QctJBgXEkbzeKz6PJy6bt3JSPNSN4I2mITYW+eKUoQ= github.com/AlekSi/gocov-xml v1.0.0/go.mod h1:J0qYeZ6tDg4oZubW9mAAgxlqw39PDfoEkzB3HXSbEuA= +github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349 h1:aq9/uAnlenI/ensOMpOSvowqzEPQhIoZ9nmadMBCKNs= +github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= @@ -170,7 +172,6 @@ github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQF github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -185,7 +186,6 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -364,9 +364,6 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMtVcOkjUcuQKh+YrluSo7+7YMCQSzy30= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= -github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26 h1:Tw4afZ2Tyr8iT8Oln6/szMjh5IDs+GtlnLsDo/Y2HEE= -github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= @@ -692,11 +689,9 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= diff --git a/pkg/storage/endpoint/keyspace.go b/pkg/storage/endpoint/keyspace.go index bc3c28c05b0..7aa82e8985b 100644 --- a/pkg/storage/endpoint/keyspace.go +++ b/pkg/storage/endpoint/keyspace.go @@ -15,17 +15,20 @@ package endpoint import ( + "context" "strconv" "github.com/gogo/protobuf/proto" "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/storage/kv" "go.etcd.io/etcd/clientv3" ) const ( - // spaceIDBase is base used to encode/decode spaceID. + // SpaceIDBase is base used to encode/decode spaceID. // It's set to 10 for better readability. - spaceIDBase = 10 + SpaceIDBase = 10 // spaceIDBitSizeMax is the max bitSize of spaceID. // It's currently set to 24 (3bytes). spaceIDBitSizeMax = 24 @@ -33,41 +36,70 @@ const ( // KeyspaceStorage defines storage operations on keyspace related data. type KeyspaceStorage interface { - // SaveKeyspace saves the given keyspace to the storage. - SaveKeyspace(*keyspacepb.KeyspaceMeta) error - // LoadKeyspace loads keyspace specified by spaceID. - LoadKeyspace(spaceID uint32, keyspace *keyspacepb.KeyspaceMeta) (bool, error) - // RemoveKeyspace removes target keyspace specified by spaceID. - RemoveKeyspace(spaceID uint32) error + SaveKeyspaceMeta(txn kv.Txn, meta *keyspacepb.KeyspaceMeta) error + LoadKeyspaceMeta(txn kv.Txn, id uint32) (*keyspacepb.KeyspaceMeta, error) + SaveKeyspaceID(txn kv.Txn, id uint32, name string) error + LoadKeyspaceID(txn kv.Txn, name string) (bool, uint32, error) // LoadRangeKeyspace loads no more than limit keyspaces starting at startID. LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) - // SaveKeyspaceIDByName saves keyspace name to ID lookup information. - // It saves the ID onto the path encoded with name. - SaveKeyspaceIDByName(spaceID uint32, name string) error - // LoadKeyspaceIDByName loads keyspace ID for the given keyspace specified by name. - // It first constructs path to spaceID with the given name, then attempt to retrieve - // target spaceID. If the target keyspace does not exist, result boolean is set to false. - LoadKeyspaceIDByName(name string) (bool, uint32, error) + RunInTxn(ctx context.Context, f func(txn kv.Txn) error) error } var _ KeyspaceStorage = (*StorageEndpoint)(nil) -// SaveKeyspace saves the given keyspace to the storage. -func (se *StorageEndpoint) SaveKeyspace(keyspace *keyspacepb.KeyspaceMeta) error { - key := KeyspaceMetaPath(keyspace.GetId()) - return se.saveProto(key, keyspace) +// SaveKeyspaceMeta adds a save keyspace meta operation to target transaction. +func (se *StorageEndpoint) SaveKeyspaceMeta(txn kv.Txn, meta *keyspacepb.KeyspaceMeta) error { + metaPath := KeyspaceMetaPath(meta.GetId()) + metaVal, err := proto.Marshal(meta) + if err != nil { + return errs.ErrProtoMarshal.Wrap(err).GenWithStackByCause() + } + return txn.Save(metaPath, string(metaVal)) +} + +// LoadKeyspaceMeta load and return keyspace meta specified by id. +// If keyspace does not exist or error occurs, returned meta will be nil. +func (se *StorageEndpoint) LoadKeyspaceMeta(txn kv.Txn, id uint32) (*keyspacepb.KeyspaceMeta, error) { + metaPath := KeyspaceMetaPath(id) + metaVal, err := txn.Load(metaPath) + if err != nil || metaVal == "" { + return nil, err + } + meta := &keyspacepb.KeyspaceMeta{} + err = proto.Unmarshal([]byte(metaVal), meta) + if err != nil { + return nil, errs.ErrProtoUnmarshal.Wrap(err).GenWithStackByCause() + } + return meta, nil +} + +// SaveKeyspaceID saves keyspace ID to the path specified by keyspace name. +func (se *StorageEndpoint) SaveKeyspaceID(txn kv.Txn, id uint32, name string) error { + idPath := KeyspaceIDPath(name) + idVal := strconv.FormatUint(uint64(id), SpaceIDBase) + return txn.Save(idPath, idVal) } -// LoadKeyspace loads keyspace specified by spaceID. -func (se *StorageEndpoint) LoadKeyspace(spaceID uint32, keyspace *keyspacepb.KeyspaceMeta) (bool, error) { - key := KeyspaceMetaPath(spaceID) - return se.loadProto(key, keyspace) +// LoadKeyspaceID loads keyspace ID from the path specified by keyspace name. +// An additional boolean is returned to indicate whether target id exists, +// it returns false if target id not found, or if error occurred. +func (se *StorageEndpoint) LoadKeyspaceID(txn kv.Txn, name string) (bool, uint32, error) { + idPath := KeyspaceIDPath(name) + idVal, err := txn.Load(idPath) + // Failed to load the keyspaceID if loading operation errored, or if keyspace does not exist. + if err != nil || idVal == "" { + return false, 0, err + } + id64, err := strconv.ParseUint(idVal, SpaceIDBase, spaceIDBitSizeMax) + if err != nil { + return false, 0, err + } + return true, uint32(id64), nil } -// RemoveKeyspace removes target keyspace specified by spaceID. -func (se *StorageEndpoint) RemoveKeyspace(spaceID uint32) error { - key := KeyspaceMetaPath(spaceID) - return se.Remove(key) +// RunInTxn runs the given function in a transaction. +func (se *StorageEndpoint) RunInTxn(ctx context.Context, f func(txn kv.Txn) error) error { + return se.Base.RunInTxn(ctx, f) } // LoadRangeKeyspace loads keyspaces starting at startID. @@ -92,25 +124,3 @@ func (se *StorageEndpoint) LoadRangeKeyspace(startID uint32, limit int) ([]*keys } return keyspaces, nil } - -// SaveKeyspaceIDByName saves keyspace name to ID lookup information to storage. -func (se *StorageEndpoint) SaveKeyspaceIDByName(spaceID uint32, name string) error { - key := KeyspaceIDPath(name) - idStr := strconv.FormatUint(uint64(spaceID), spaceIDBase) - return se.Save(key, idStr) -} - -// LoadKeyspaceIDByName loads keyspace ID for the given keyspace name -func (se *StorageEndpoint) LoadKeyspaceIDByName(name string) (bool, uint32, error) { - key := KeyspaceIDPath(name) - idStr, err := se.Load(key) - // Failed to load the keyspaceID if loading operation errored, or if keyspace does not exist. - if err != nil || idStr == "" { - return false, 0, err - } - id64, err := strconv.ParseUint(idStr, spaceIDBase, spaceIDBitSizeMax) - if err != nil { - return false, 0, err - } - return true, uint32(id64), nil -} diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index 2841e5c568d..bbf9b1e7fdc 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -15,13 +15,18 @@ package keyspace import ( + "context" "time" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/pingcap/log" "github.com/tikv/pd/pkg/id" + "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/syncutil" + "go.uber.org/zap" ) const ( @@ -39,14 +44,14 @@ const ( // Manager manages keyspace related data. // It validates requests and provides concurrency control. type Manager struct { - // idLock guards keyspace name to id lookup entries. - idLock syncutil.Mutex // metaLock guards keyspace meta. metaLock *syncutil.LockGroup // idAllocator allocates keyspace id. idAllocator id.Allocator // store is the storage for keyspace related information. store endpoint.KeyspaceStorage + // ctx is the context of the manager, to be used in transaction. + ctx context.Context } // CreateKeyspaceRequest represents necessary arguments to create a keyspace. @@ -60,21 +65,17 @@ type CreateKeyspaceRequest struct { } // NewKeyspaceManager creates a Manager of keyspace related data. -func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator) (*Manager, error) { - manager := &Manager{ - store: store, +func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator) *Manager { + return &Manager{ + metaLock: syncutil.NewLockGroup(syncutil.WithHash(keyspaceIDHash)), idAllocator: idAllocator, - metaLock: syncutil.NewLockGroup(syncutil.WithHash(SpaceIDHash)), - } - // If default keyspace already exists, skip initialization. - defaultExist, _, err := manager.store.LoadKeyspaceIDByName(DefaultKeyspaceName) - if err != nil { - return nil, err - } - if defaultExist { - return manager, nil + store: store, + ctx: context.TODO(), } - // Initialize default keyspace. +} + +// Bootstrap saves default keyspace info. +func (manager *Manager) Bootstrap() error { now := time.Now().Unix() defaultKeyspace := &keyspacepb.KeyspaceMeta{ Id: DefaultKeyspaceID, @@ -83,11 +84,13 @@ func NewKeyspaceManager(store endpoint.KeyspaceStorage, idAllocator id.Allocator CreatedAt: now, StateChangedAt: now, } - _, err = manager.saveNewKeyspace(defaultKeyspace) + err := manager.saveNewKeyspace(defaultKeyspace) + // It's possible that default keyspace already exists in the storage (e.g. PD restart/recover), + // so we ignore the keyspaceExists error. if err != nil && err != ErrKeyspaceExists { - return nil, err + return err } - return manager, nil + return nil } // CreateKeyspace create a keyspace meta with given config and save it to storage. @@ -110,72 +113,95 @@ func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspac StateChangedAt: request.Now, Config: request.Config, } - return manager.saveNewKeyspace(keyspace) -} - -func (manager *Manager) saveNewKeyspace(keyspace *keyspacepb.KeyspaceMeta) (*keyspacepb.KeyspaceMeta, error) { - manager.idLock.Lock() - defer manager.idLock.Unlock() - // Check if keyspace id with that name already exists. - nameExists, _, err := manager.store.LoadKeyspaceIDByName(keyspace.GetName()) - if err != nil { - return nil, err - } - if nameExists { - return nil, ErrKeyspaceExists - } - manager.metaLock.Lock(keyspace.GetId()) - defer manager.metaLock.Unlock(keyspace.GetId()) - // Check if keyspace meta with that id already exists. - keyspaceExists, err := manager.store.LoadKeyspace(keyspace.GetId(), &keyspacepb.KeyspaceMeta{}) + err = manager.saveNewKeyspace(keyspace) if err != nil { + log.Warn("[keyspace] failed to create keyspace", + zap.Uint32("ID", keyspace.GetId()), + zap.String("name", keyspace.GetName()), + zap.Error(err), + ) return nil, err } - if keyspaceExists { - return nil, ErrKeyspaceExists - } - // TODO: Enable Transaction at storage layer to save MetaData and NameToID in a single transaction. - // Save keyspace meta before saving id. - if err = manager.store.SaveKeyspace(keyspace); err != nil { - return nil, err - } - // Create name to ID entry, - // if this failed, previously stored keyspace meta should be removed. - if err = manager.createNameToID(keyspace.GetId(), keyspace.GetName()); err != nil { - if removeErr := manager.store.RemoveKeyspace(keyspace.GetId()); removeErr != nil { - return nil, errors.Wrap(removeErr, "failed to remove keyspace keyspace after save spaceID failure") - } - return nil, err - } - + log.Info("[keyspace] keyspace created", + zap.Uint32("ID", keyspace.GetId()), + zap.String("name", keyspace.GetName()), + ) return keyspace, nil } +func (manager *Manager) saveNewKeyspace(keyspace *keyspacepb.KeyspaceMeta) error { + manager.metaLock.Lock(keyspace.Id) + defer manager.metaLock.Unlock(keyspace.Id) + + return manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + // Save keyspace ID. + // Check if keyspace with that name already exists. + nameExists, _, err := manager.store.LoadKeyspaceID(txn, keyspace.Name) + if err != nil { + return err + } + if nameExists { + return ErrKeyspaceExists + } + err = manager.store.SaveKeyspaceID(txn, keyspace.Id, keyspace.Name) + if err != nil { + return err + } + // Save keyspace meta. + // Check if keyspace with that id already exists. + loadedMeta, err := manager.store.LoadKeyspaceMeta(txn, keyspace.Id) + if err != nil { + return err + } + if loadedMeta != nil { + return ErrKeyspaceExists + } + return manager.store.SaveKeyspaceMeta(txn, keyspace) + }) +} + // LoadKeyspace returns the keyspace specified by name. // It returns error if loading or unmarshalling met error or if keyspace does not exist. func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, error) { - // First get keyspace ID from the name given. - loaded, spaceID, err := manager.store.LoadKeyspaceIDByName(name) - if err != nil { - return nil, err - } - if !loaded { - return nil, ErrKeyspaceNotFound - } - return manager.loadKeyspaceByID(spaceID) + var meta *keyspacepb.KeyspaceMeta + err := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + loaded, id, err := manager.store.LoadKeyspaceID(txn, name) + if err != nil { + return err + } + if !loaded { + return ErrKeyspaceNotFound + } + meta, err = manager.store.LoadKeyspaceMeta(txn, id) + if err != nil { + return err + } + if meta == nil { + return ErrKeyspaceNotFound + } + return nil + }) + return meta, err } -func (manager *Manager) loadKeyspaceByID(spaceID uint32) (*keyspacepb.KeyspaceMeta, error) { - // Load the keyspace with target ID. - keyspace := &keyspacepb.KeyspaceMeta{} - loaded, err := manager.store.LoadKeyspace(spaceID, keyspace) - if err != nil { - return nil, err - } - if !loaded { - return nil, ErrKeyspaceNotFound - } - return keyspace, nil +// LoadKeyspaceByID returns the keyspace specified by id. +// It returns error if loading or unmarshalling met error or if keyspace does not exist. +func (manager *Manager) LoadKeyspaceByID(id uint32) (*keyspacepb.KeyspaceMeta, error) { + var ( + meta *keyspacepb.KeyspaceMeta + err error + ) + err = manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + meta, err = manager.store.LoadKeyspaceMeta(txn, id) + if err != nil { + return err + } + if meta == nil { + return ErrKeyspaceNotFound + } + return nil + }) + return meta, err } // Mutation represents a single operation to be applied on keyspace config. @@ -202,44 +228,63 @@ const ( // UpdateKeyspaceConfig changes target keyspace's config in the order specified in mutations. // It returns error if saving failed, operation not allowed, or if keyspace not exists. func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation) (*keyspacepb.KeyspaceMeta, error) { - // First get KeyspaceID from Name. - loaded, spaceID, err := manager.store.LoadKeyspaceIDByName(name) - if err != nil { - return nil, err - } - if !loaded { - return nil, ErrKeyspaceNotFound - } - manager.metaLock.Lock(spaceID) - defer manager.metaLock.Unlock(spaceID) - // Load keyspace by id. - keyspace, err := manager.loadKeyspaceByID(spaceID) - if err != nil { - return nil, err - } - // Changing ARCHIVED keyspace's config is not allowed. - if keyspace.GetState() == keyspacepb.KeyspaceState_ARCHIVED { - return nil, errKeyspaceArchived - } - if keyspace.GetConfig() == nil { - keyspace.Config = map[string]string{} - } - // Update keyspace config according to mutations. - for _, mutation := range mutations { - switch mutation.Op { - case OpPut: - keyspace.Config[mutation.Key] = mutation.Value - case OpDel: - delete(keyspace.Config, mutation.Key) - default: - return nil, errIllegalOperation + var meta *keyspacepb.KeyspaceMeta + err := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + // First get KeyspaceID from Name. + loaded, id, err := manager.store.LoadKeyspaceID(txn, name) + if err != nil { + return err } - } - // Save the updated keyspace. - if err = manager.store.SaveKeyspace(keyspace); err != nil { + if !loaded { + return ErrKeyspaceNotFound + } + manager.metaLock.Lock(id) + defer manager.metaLock.Unlock(id) + // Load keyspace by id. + meta, err = manager.store.LoadKeyspaceMeta(txn, id) + if err != nil { + return err + } + if meta == nil { + return ErrKeyspaceNotFound + } + // Only keyspace with state listed in allowChangeConfig are allowed to change their config. + if !slice.Contains(allowChangeConfig, meta.GetState()) { + return errors.Errorf("cannot change config for keyspace with state %s", meta.GetState().String()) + } + // Initialize meta's config map if it's nil. + if meta.GetConfig() == nil { + meta.Config = map[string]string{} + } + // Update keyspace config according to mutations. + for _, mutation := range mutations { + switch mutation.Op { + case OpPut: + meta.Config[mutation.Key] = mutation.Value + case OpDel: + delete(meta.Config, mutation.Key) + default: + return errIllegalOperation + } + } + // Save the updated keyspace meta. + return manager.store.SaveKeyspaceMeta(txn, meta) + }) + + if err != nil { + log.Warn("[keyspace] failed to update keyspace config", + zap.Uint32("ID", meta.GetId()), + zap.String("name", meta.GetName()), + zap.Error(err), + ) return nil, err } - return keyspace, nil + log.Info("[keyspace] keyspace config updated", + zap.Uint32("ID", meta.GetId()), + zap.String("name", meta.GetName()), + zap.Any("new config", meta.GetConfig()), + ) + return meta, nil } // UpdateKeyspaceState updates target keyspace to the given state if it's not already in that state. @@ -247,43 +292,112 @@ func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation) func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.KeyspaceState, now int64) (*keyspacepb.KeyspaceMeta, error) { // Changing the state of default keyspace is not allowed. if name == DefaultKeyspaceName { + log.Warn("[keyspace] failed to update keyspace config", + zap.Error(errModifyDefault), + ) return nil, errModifyDefault } - // First get KeyspaceID from Name. - loaded, spaceID, err := manager.store.LoadKeyspaceIDByName(name) + var meta *keyspacepb.KeyspaceMeta + err := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + // First get KeyspaceID from Name. + loaded, id, err := manager.store.LoadKeyspaceID(txn, name) + if err != nil { + return err + } + if !loaded { + return ErrKeyspaceNotFound + } + manager.metaLock.Lock(id) + defer manager.metaLock.Unlock(id) + // Load keyspace by id. + meta, err = manager.store.LoadKeyspaceMeta(txn, id) + if err != nil { + return err + } + if meta == nil { + return ErrKeyspaceNotFound + } + // Update keyspace meta. + if err = updateKeyspaceState(meta, newState, now); err != nil { + return err + } + return manager.store.SaveKeyspaceMeta(txn, meta) + }) if err != nil { + log.Warn("[keyspace] failed to update keyspace config", + zap.Uint32("ID", meta.GetId()), + zap.String("name", meta.GetName()), + zap.Error(err), + ) return nil, err } - if !loaded { - return nil, ErrKeyspaceNotFound + log.Info("[keyspace] keyspace state updated", + zap.Uint32("ID", meta.GetId()), + zap.String("name", meta.GetName()), + zap.String("new state", newState.String()), + ) + return meta, nil +} + +// UpdateKeyspaceStateByID updates target keyspace to the given state if it's not already in that state. +// It returns error if saving failed, operation not allowed, or if keyspace not exists. +func (manager *Manager) UpdateKeyspaceStateByID(id uint32, newState keyspacepb.KeyspaceState, now int64) (*keyspacepb.KeyspaceMeta, error) { + // Changing the state of default keyspace is not allowed. + if id == DefaultKeyspaceID { + log.Warn("[keyspace] failed to update keyspace config", + zap.Error(errModifyDefault), + ) + return nil, errModifyDefault } - manager.metaLock.Lock(spaceID) - defer manager.metaLock.Unlock(spaceID) - // Load keyspace by id. - keyspace, err := manager.loadKeyspaceByID(spaceID) + var meta *keyspacepb.KeyspaceMeta + var err error + err = manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + manager.metaLock.Lock(id) + defer manager.metaLock.Unlock(id) + // Load keyspace by id. + meta, err = manager.store.LoadKeyspaceMeta(txn, id) + if err != nil { + return err + } + if meta == nil { + return ErrKeyspaceNotFound + } + // Update keyspace meta. + if err = updateKeyspaceState(meta, newState, now); err != nil { + return err + } + return manager.store.SaveKeyspaceMeta(txn, meta) + }) if err != nil { + log.Warn("[keyspace] failed to update keyspace config", + zap.Uint32("ID", meta.GetId()), + zap.String("name", meta.GetName()), + zap.Error(err), + ) return nil, err } - // If keyspace is already in target state, then nothing needs to be change. - if keyspace.GetState() == newState { - return keyspace, nil - } - // ARCHIVED is the terminal state that cannot be changed from. - if keyspace.GetState() == keyspacepb.KeyspaceState_ARCHIVED { - return nil, errKeyspaceArchived - } - // Archiving an enabled keyspace directly is not allowed. - if keyspace.GetState() == keyspacepb.KeyspaceState_ENABLED && newState == keyspacepb.KeyspaceState_ARCHIVED { - return nil, errArchiveEnabled - } - // Change keyspace state and record change time. - keyspace.StateChangedAt = now - keyspace.State = newState - // Save the updated keyspace. - if err = manager.store.SaveKeyspace(keyspace); err != nil { - return nil, err - } - return keyspace, nil + log.Info("[keyspace] keyspace state updated", + zap.Uint32("ID", meta.GetId()), + zap.String("name", meta.GetName()), + zap.String("new state", newState.String()), + ) + return meta, nil +} + +// updateKeyspaceState updates keyspace meta and record the update time. +func updateKeyspaceState(meta *keyspacepb.KeyspaceMeta, newState keyspacepb.KeyspaceState, now int64) error { + // If already in the target state, do nothing and return. + if meta.GetState() == newState { + return nil + } + // Consult state transition table to check if the operation is legal. + if !slice.Contains(stateTransitionTable[meta.GetState()], newState) { + return errors.Errorf("cannot change keyspace state from %s to %s", meta.GetState().String(), newState.String()) + } + // If the operation is legal, update keyspace state and change time. + meta.State = newState + meta.StateChangedAt = now + return nil } // LoadRangeKeyspace load up to limit keyspaces starting from keyspace with startID. @@ -307,9 +421,3 @@ func (manager *Manager) allocID() (uint32, error) { } return id32, nil } - -// createNameToID create a keyspace name to ID lookup entry. -// It returns error if saving keyspace name meet error. -func (manager *Manager) createNameToID(spaceID uint32, name string) error { - return manager.store.SaveKeyspaceIDByName(spaceID, name) -} diff --git a/server/keyspace/keyspace_test.go b/server/keyspace/keyspace_test.go index 31ea0930c1b..56f2066c15a 100644 --- a/server/keyspace/keyspace_test.go +++ b/server/keyspace/keyspace_test.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" @@ -35,12 +36,20 @@ const ( testConfig2 = "config_entry_2" ) -func mustNewKeyspaceManager(re *require.Assertions) *Manager { +type keyspaceTestSuite struct { + suite.Suite + manager *Manager +} + +func TestKeyspaceTestSuite(t *testing.T) { + suite.Run(t, new(keyspaceTestSuite)) +} + +func (suite *keyspaceTestSuite) SetupTest() { store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) allocator := mockid.NewIDAllocator() - manager, err := NewKeyspaceManager(store, allocator) - re.NoError(err) - return manager + suite.manager = NewKeyspaceManager(store, allocator) + suite.NoError(suite.manager.Bootstrap()) } func makeCreateKeyspaceRequests(count int) []*CreateKeyspaceRequest { @@ -59,9 +68,9 @@ func makeCreateKeyspaceRequests(count int) []*CreateKeyspaceRequest { return requests } -func TestCreateKeyspace(t *testing.T) { - re := require.New(t) - manager := mustNewKeyspaceManager(re) +func (suite *keyspaceTestSuite) TestCreateKeyspace() { + re := suite.Require() + manager := suite.manager requests := makeCreateKeyspaceRequests(10) for i, request := range requests { @@ -104,9 +113,9 @@ func makeMutations() []*Mutation { } } -func TestUpdateKeyspaceConfig(t *testing.T) { - re := require.New(t) - manager := mustNewKeyspaceManager(re) +func (suite *keyspaceTestSuite) TestUpdateKeyspaceConfig() { + re := suite.Require() + manager := suite.manager requests := makeCreateKeyspaceRequests(5) mutations := makeMutations() for _, createRequest := range requests { @@ -116,9 +125,9 @@ func TestUpdateKeyspaceConfig(t *testing.T) { re.NoError(err) checkMutations(re, createRequest.Config, updated.Config, mutations) // Changing config of a ARCHIVED keyspace is not allowed. - _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_DISABLED, 0) + _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_DISABLED, time.Now().Unix()) re.NoError(err) - _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ARCHIVED, 0) + _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ARCHIVED, time.Now().Unix()) re.NoError(err) _, err = manager.UpdateKeyspaceConfig(createRequest.Name, mutations) re.Error(err) @@ -129,9 +138,9 @@ func TestUpdateKeyspaceConfig(t *testing.T) { checkMutations(re, nil, updated.Config, mutations) } -func TestUpdateKeyspaceState(t *testing.T) { - re := require.New(t) - manager := mustNewKeyspaceManager(re) +func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { + re := suite.Require() + manager := suite.manager requests := makeCreateKeyspaceRequests(5) for _, createRequest := range requests { _, err := manager.CreateKeyspace(createRequest) @@ -166,9 +175,9 @@ func TestUpdateKeyspaceState(t *testing.T) { } } -func TestLoadRangeKeyspace(t *testing.T) { - re := require.New(t) - manager := mustNewKeyspaceManager(re) +func (suite *keyspaceTestSuite) TestLoadRangeKeyspace() { + re := suite.Require() + manager := suite.manager // Test with 100 keyspaces. // Created keyspace ids are 1 - 100. total := 100 @@ -238,9 +247,9 @@ func TestLoadRangeKeyspace(t *testing.T) { // TestUpdateMultipleKeyspace checks that updating multiple keyspace's config simultaneously // will be successful. -func TestUpdateMultipleKeyspace(t *testing.T) { - re := require.New(t) - manager := mustNewKeyspaceManager(re) +func (suite *keyspaceTestSuite) TestUpdateMultipleKeyspace() { + re := suite.Require() + manager := suite.manager requests := makeCreateKeyspaceRequests(50) for _, createRequest := range requests { _, err := manager.CreateKeyspace(createRequest) diff --git a/server/keyspace/util.go b/server/keyspace/util.go index 666052e6dad..a12babe7e4a 100644 --- a/server/keyspace/util.go +++ b/server/keyspace/util.go @@ -18,6 +18,7 @@ import ( "regexp" "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/keyspacepb" ) const ( @@ -33,21 +34,30 @@ var ( // ErrKeyspaceExists indicates target keyspace already exists. // Used when creating a new keyspace. ErrKeyspaceExists = errors.New("keyspace already exists") - errKeyspaceArchived = errors.New("keyspace already archived") - errArchiveEnabled = errors.New("cannot archive ENABLED keyspace") errModifyDefault = errors.New("cannot modify default keyspace's state") errIllegalOperation = errors.New("unknown operation") + + // stateTransitionTable lists all allowed next state for the given current state. + // Note that transit from any state to itself is allowed for idempotence. + stateTransitionTable = map[keyspacepb.KeyspaceState][]keyspacepb.KeyspaceState{ + keyspacepb.KeyspaceState_ENABLED: {keyspacepb.KeyspaceState_ENABLED, keyspacepb.KeyspaceState_DISABLED}, + keyspacepb.KeyspaceState_DISABLED: {keyspacepb.KeyspaceState_DISABLED, keyspacepb.KeyspaceState_ENABLED, keyspacepb.KeyspaceState_ARCHIVED}, + keyspacepb.KeyspaceState_ARCHIVED: {keyspacepb.KeyspaceState_ARCHIVED, keyspacepb.KeyspaceState_TOMBSTONE}, + keyspacepb.KeyspaceState_TOMBSTONE: {keyspacepb.KeyspaceState_TOMBSTONE}, + } + // Only keyspaces in the state specified by allowChangeConfig are allowed to change their config. + allowChangeConfig = []keyspacepb.KeyspaceState{keyspacepb.KeyspaceState_ENABLED, keyspacepb.KeyspaceState_DISABLED} ) // validateID check if keyspace falls within the acceptable range. // It throws errIllegalID when input id is our of range, // or if it collides with reserved id. -func validateID(spaceID uint32) error { - if spaceID > spaceIDMax { - return errors.Errorf("illegal keyspace id %d, larger than spaceID Max %d", spaceID, spaceIDMax) +func validateID(id uint32) error { + if id > spaceIDMax { + return errors.Errorf("illegal keyspace id %d, larger than spaceID Max %d", id, spaceIDMax) } - if spaceID == DefaultKeyspaceID { - return errors.Errorf("illegal keyspace id %d, collides with default keyspace id", spaceID) + if id == DefaultKeyspaceID { + return errors.Errorf("illegal keyspace id %d, collides with default keyspace id", id) } return nil } @@ -69,11 +79,11 @@ func validateName(name string) error { return nil } -// SpaceIDHash is used to hash the spaceID inside the lockGroup. +// keyspaceIDHash is used to hash the spaceID inside the lockGroup. // A simple mask is applied to spaceID to use its last byte as map key, // limiting the maximum map length to 256. // Since keyspaceID is sequentially allocated, this can also reduce the chance // of collision when comparing with random hashes. -func SpaceIDHash(spaceID uint32) uint32 { - return spaceID & 0xFF +func keyspaceIDHash(id uint32) uint32 { + return id & 0xFF } diff --git a/server/keyspace_service.go b/server/keyspace_service.go index fefc2315a8d..5fb06c38bdb 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -17,6 +17,7 @@ package server import ( "context" "path" + "time" "github.com/gogo/protobuf/proto" "github.com/pingcap/kvproto/pkg/keyspacepb" @@ -57,7 +58,7 @@ func (s *KeyspaceServer) LoadKeyspace(_ context.Context, request *keyspacepb.Loa } manager := s.GetKeyspaceManager() - meta, err := manager.LoadKeyspace(request.Name) + meta, err := manager.LoadKeyspace(request.GetName()) if err != nil { return &keyspacepb.LoadKeyspaceResponse{Header: s.getErrorHeader(err)}, nil } @@ -126,3 +127,23 @@ func (s *KeyspaceServer) sendAllKeyspaceMeta(ctx context.Context, stream keyspac } return stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.header(), Keyspaces: metas}) } + +// UpdateKeyspaceState updates the state of keyspace specified in the request. +func (s *KeyspaceServer) UpdateKeyspaceState(_ context.Context, request *keyspacepb.UpdateKeyspaceStateRequest) (*keyspacepb.UpdateKeyspaceStateResponse, error) { + if err := s.validateRequest(request.GetHeader()); err != nil { + return nil, err + } + rc := s.GetRaftCluster() + if rc == nil { + return &keyspacepb.UpdateKeyspaceStateResponse{Header: s.notBootstrappedHeader()}, nil + } + manager := s.GetKeyspaceManager() + meta, err := manager.UpdateKeyspaceStateByID(request.GetId(), request.GetState(), time.Now().Unix()) + if err != nil { + return &keyspacepb.UpdateKeyspaceStateResponse{Header: s.getErrorHeader(err)}, nil + } + return &keyspacepb.UpdateKeyspaceStateResponse{ + Header: s.header(), + Keyspace: meta, + }, nil +} From dfaa8cd55fa0a5be2021a1e420fedc0c725f7e8c Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 9 Jan 2023 20:07:14 +0800 Subject: [PATCH 70/76] fix test Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/server.go | 9 ++-- tests/client/go.mod | 3 ++ tests/client/go.sum | 10 +---- tests/client/keyspace_test.go | 84 ++++++++++++++++++++++++++++++++--- 4 files changed, 87 insertions(+), 19 deletions(-) diff --git a/server/server.go b/server/server.go index 84fbedd6948..66b45512cc7 100644 --- a/server/server.go +++ b/server/server.go @@ -382,10 +382,7 @@ func (s *Server) startServer(ctx context.Context) error { Member: s.member.MemberValue(), Step: keyspace.AllocStep, }) - s.keyspaceManager, err = keyspace.NewKeyspaceManager(s.storage, keyspaceIDAllocator) - if err != nil { - return err - } + s.keyspaceManager = keyspace.NewKeyspaceManager(s.storage, keyspaceIDAllocator) s.basicCluster = core.NewBasicCluster() s.cluster = cluster.NewRaftCluster(ctx, s.clusterID, syncer.NewRegionSyncer(s), s.client, s.httpClient) s.hbStreams = hbstream.NewHeartbeatStreams(ctx, s.clusterID, s.cluster) @@ -647,6 +644,10 @@ func (s *Server) bootstrapCluster(req *pdpb.BootstrapRequest) (*pdpb.BootstrapRe log.Warn("flush the bootstrap region failed", errs.ZapError(err)) } + if err := s.GetKeyspaceManager().Bootstrap(); err != nil { + log.Warn("bootstrap keyspace manager failed") + } + if err := s.cluster.Start(s); err != nil { return nil, err } diff --git a/tests/client/go.mod b/tests/client/go.mod index 85ca22dd688..906601f8a20 100644 --- a/tests/client/go.mod +++ b/tests/client/go.mod @@ -166,3 +166,6 @@ replace ( google.golang.org/grpc v1.43.0 => google.golang.org/grpc v1.26.0 google.golang.org/protobuf v1.26.0 => github.com/golang/protobuf v1.3.4 ) + +// TODO: remove after kvproto merge +replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349 diff --git a/tests/client/go.sum b/tests/client/go.sum index ab7065da67b..9453c1feb4c 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349 h1:aq9/uAnlenI/ensOMpOSvowqzEPQhIoZ9nmadMBCKNs= +github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= @@ -145,7 +147,6 @@ github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQF github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -159,7 +160,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -326,10 +326,6 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26 h1:Tw4afZ2Tyr8iT8Oln6/szMjh5IDs+GtlnLsDo/Y2HEE= -github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= @@ -628,11 +624,9 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= diff --git a/tests/client/keyspace_test.go b/tests/client/keyspace_test.go index 63355620de7..a9fb953cdf5 100644 --- a/tests/client/keyspace_test.go +++ b/tests/client/keyspace_test.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/server" "github.com/tikv/pd/server/keyspace" ) @@ -52,7 +53,7 @@ func (suite *clientTestSuite) TestLoadKeyspace() { re := suite.Require() metas := mustMakeTestKeyspaces(re, suite.srv, 0, 10) for _, expected := range metas { - loaded, err := suite.client.LoadKeyspace(suite.ctx, expected.Name) + loaded, err := suite.client.LoadKeyspace(suite.ctx, expected.GetName()) re.NoError(err) re.Equal(expected, loaded) } @@ -62,22 +63,20 @@ func (suite *clientTestSuite) TestLoadKeyspace() { // Loading default keyspace should be successful. keyspaceDefault, err := suite.client.LoadKeyspace(suite.ctx, keyspace.DefaultKeyspaceName) re.NoError(err) - re.Equal(keyspace.DefaultKeyspaceID, keyspaceDefault.Id) - re.Equal(keyspace.DefaultKeyspaceName, keyspaceDefault.Name) + re.Equal(keyspace.DefaultKeyspaceID, keyspaceDefault.GetId()) + re.Equal(keyspace.DefaultKeyspaceName, keyspaceDefault.GetName()) } -func (suite *clientTestSuite) TestWatchKeyspace() { +func (suite *clientTestSuite) TestWatchKeyspaces() { re := suite.Require() initialKeyspaces := mustMakeTestKeyspaces(re, suite.srv, 10, 10) watchChan, err := suite.client.WatchKeyspaces(suite.ctx) re.NoError(err) - // First batch of watchChan message should contain all existing keyspaces, including the default. + // First batch of watchChan message should contain all existing keyspaces. initialLoaded := <-watchChan for i := range initialKeyspaces { re.Contains(initialLoaded, initialKeyspaces[i]) } - keyspaceDefault, err := suite.client.LoadKeyspace(suite.ctx, keyspace.DefaultKeyspaceName) - re.Contains(initialLoaded, keyspaceDefault) // Each additional message contains extra put events. additionalKeyspaces := mustMakeTestKeyspaces(re, suite.srv, 30, 10) re.NoError(err) @@ -117,3 +116,74 @@ func (suite *clientTestSuite) TestWatchKeyspace() { loaded = <-watchChan re.Equal([]*keyspacepb.KeyspaceMeta{expected}, loaded) } + +func mustCreateKeyspaceAtState(re *require.Assertions, server *server.Server, index int, state keyspacepb.KeyspaceState) *keyspacepb.KeyspaceMeta { + manager := server.GetKeyspaceManager() + meta, err := manager.CreateKeyspace(&keyspace.CreateKeyspaceRequest{ + Name: fmt.Sprintf("test_keyspace%d", index), + Config: nil, + Now: 0, // Use 0 to indicate unchanged keyspace. + }) + re.NoError(err) + switch state { + case keyspacepb.KeyspaceState_ENABLED: + case keyspacepb.KeyspaceState_DISABLED: + meta, err = manager.UpdateKeyspaceStateByID(meta.GetId(), keyspacepb.KeyspaceState_DISABLED, 0) + re.NoError(err) + case keyspacepb.KeyspaceState_ARCHIVED: + meta, err = manager.UpdateKeyspaceStateByID(meta.GetId(), keyspacepb.KeyspaceState_DISABLED, 0) + re.NoError(err) + meta, err = manager.UpdateKeyspaceStateByID(meta.GetId(), keyspacepb.KeyspaceState_ARCHIVED, 0) + re.NoError(err) + case keyspacepb.KeyspaceState_TOMBSTONE: + meta, err = manager.UpdateKeyspaceStateByID(meta.GetId(), keyspacepb.KeyspaceState_DISABLED, 0) + re.NoError(err) + meta, err = manager.UpdateKeyspaceStateByID(meta.GetId(), keyspacepb.KeyspaceState_ARCHIVED, 0) + re.NoError(err) + meta, err = manager.UpdateKeyspaceStateByID(meta.GetId(), keyspacepb.KeyspaceState_TOMBSTONE, 0) + re.NoError(err) + default: + re.Fail("unknown keyspace state") + } + return meta +} + +func (suite *clientTestSuite) TestUpdateKeyspaceState() { + re := suite.Require() + allStates := []keyspacepb.KeyspaceState{ + keyspacepb.KeyspaceState_ENABLED, + keyspacepb.KeyspaceState_DISABLED, + keyspacepb.KeyspaceState_ARCHIVED, + keyspacepb.KeyspaceState_TOMBSTONE, + } + allowedTransitions := map[keyspacepb.KeyspaceState][]keyspacepb.KeyspaceState{ + keyspacepb.KeyspaceState_ENABLED: {keyspacepb.KeyspaceState_ENABLED, keyspacepb.KeyspaceState_DISABLED}, + keyspacepb.KeyspaceState_DISABLED: {keyspacepb.KeyspaceState_DISABLED, keyspacepb.KeyspaceState_ENABLED, keyspacepb.KeyspaceState_ARCHIVED}, + keyspacepb.KeyspaceState_ARCHIVED: {keyspacepb.KeyspaceState_ARCHIVED, keyspacepb.KeyspaceState_TOMBSTONE}, + keyspacepb.KeyspaceState_TOMBSTONE: {keyspacepb.KeyspaceState_TOMBSTONE}, + } + // Use index to avoid collision with other tests. + index := 1000 + for _, originState := range allStates { + for _, targetState := range allStates { + meta := mustCreateKeyspaceAtState(re, suite.srv, index, originState) + updated, err := suite.client.UpdateKeyspaceState(suite.ctx, meta.GetId(), targetState) + if slice.Contains(allowedTransitions[originState], targetState) { + // If transition is allowed, then update must be successful. + re.NoError(err) + if originState != targetState { + // If changing state, then must record stateChangedAt. + re.NotEqual(updated.GetStateChangedAt(), meta.GetStateChangedAt()) + } else { + // Otherwise the request should be idempotent. + re.Equal(updated, meta) + } + } else { + // If operation is not allowed, then update must fail, returned meta must be nil. + re.Error(err) + re.Nil(updated) + } + index++ + } + } +} From b65de48db1e98d19df2fab518451c25ed18552e2 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 9 Jan 2023 20:38:14 +0800 Subject: [PATCH 71/76] fix test Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/keyspace_test.go | 89 +++++++++++++++++---------------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/server/storage/keyspace_test.go b/server/storage/keyspace_test.go index 7d469dd06d7..152d71afb5c 100644 --- a/server/storage/keyspace_test.go +++ b/server/storage/keyspace_test.go @@ -15,49 +15,73 @@ package storage import ( + "context" "testing" "time" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" ) func TestSaveLoadKeyspace(t *testing.T) { re := require.New(t) storage := NewStorageWithMemoryBackend() - + // Store test keyspace id and meta. keyspaces := makeTestKeyspaces() - for _, keyspace := range keyspaces { - re.NoError(storage.SaveKeyspace(keyspace)) - } + err := storage.RunInTxn(context.TODO(), func(txn kv.Txn) error { + for _, keyspace := range keyspaces { + re.NoError(storage.SaveKeyspaceID(txn, keyspace.Id, keyspace.Name)) + re.NoError(storage.SaveKeyspaceMeta(txn, keyspace)) + } + return nil + }) + re.NoError(err) + // Test load keyspace id and meta + err = storage.RunInTxn(context.TODO(), func(txn kv.Txn) error { + for _, expectedMeta := range keyspaces { + loadSuccess, id, err := storage.LoadKeyspaceID(txn, expectedMeta.Name) + re.NoError(err) + re.True(loadSuccess) + re.Equal(expectedMeta.Id, id) + // Test load keyspace. + loadedMeta, err := storage.LoadKeyspaceMeta(txn, expectedMeta.Id) + re.NoError(err) + re.Equal(expectedMeta, loadedMeta) + } + return nil + }) + re.NoError(err) - for _, keyspace := range keyspaces { - spaceID := keyspace.GetId() - loadedKeyspace := &keyspacepb.KeyspaceMeta{} - // Test load keyspace. - success, err := storage.LoadKeyspace(spaceID, loadedKeyspace) - re.True(success) + err = storage.RunInTxn(context.TODO(), func(txn kv.Txn) error { + // Loading a non-existing keyspace id should be unsuccessful but no error. + loadSuccess, id, err := storage.LoadKeyspaceID(txn, "non-existing keyspace") re.NoError(err) - re.Equal(keyspace, loadedKeyspace) - // Test remove keyspace. - re.NoError(storage.RemoveKeyspace(spaceID)) - success, err = storage.LoadKeyspace(spaceID, loadedKeyspace) - // Loading a non-existing keyspace should be unsuccessful. - re.False(success) - // Loading a non-existing keyspace should not return error. + re.False(loadSuccess) + re.Zero(id) + // Loading a non-existing keyspace meta should be unsuccessful but no error. + meta, err := storage.LoadKeyspaceMeta(txn, 999) re.NoError(err) - } + re.Nil(meta) + return nil + }) + re.NoError(err) } func TestLoadRangeKeyspaces(t *testing.T) { re := require.New(t) storage := NewStorageWithMemoryBackend() + // Store test keyspace meta. keyspaces := makeTestKeyspaces() - for _, keyspace := range keyspaces { - re.NoError(storage.SaveKeyspace(keyspace)) - } + err := storage.RunInTxn(context.TODO(), func(txn kv.Txn) error { + for _, keyspace := range keyspaces { + re.NoError(storage.SaveKeyspaceMeta(txn, keyspace)) + } + return nil + }) + re.NoError(err) // Load all keyspaces. loadedKeyspaces, err := storage.LoadRangeKeyspace(keyspaces[0].GetId(), 0) @@ -75,29 +99,6 @@ func TestLoadRangeKeyspaces(t *testing.T) { re.ElementsMatch(keyspaces[:1], loadedKeyspace3) } -func TestSaveLoadKeyspaceID(t *testing.T) { - re := require.New(t) - storage := NewStorageWithMemoryBackend() - - ids := []uint32{100, 200, 300} - names := []string{"keyspace1", "keyspace2", "keyspace3"} - for i := range ids { - re.NoError(storage.SaveKeyspaceIDByName(ids[i], names[i])) - } - - for i := range names { - success, id, err := storage.LoadKeyspaceIDByName(names[i]) - re.NoError(err) - re.True(success) - re.Equal(ids[i], id) - } - // Loading non-existing id should return false, 0, nil. - success, id, err := storage.LoadKeyspaceIDByName("non-existing") - re.NoError(err) - re.False(success) - re.Equal(uint32(0), id) -} - func makeTestKeyspaces() []*keyspacepb.KeyspaceMeta { now := time.Now().Unix() return []*keyspacepb.KeyspaceMeta{ From c6e1a58b01fe0087a5fd94ee90018f8e58cd17ed Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 10 Jan 2023 00:58:59 +0800 Subject: [PATCH 72/76] fix test Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 3 +- tests/server/apiv2/handlers/keyspace_test.go | 37 +++++++++++--------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index 5e7321faf73..ebeb401117c 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -18,6 +18,7 @@ import ( "encoding/json" "net/http" "strconv" + "strings" "time" "github.com/gin-gonic/gin" @@ -278,7 +279,7 @@ func UpdateKeyspaceState(c *gin.Context) { c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) return } - targetState, ok := keyspacepb.KeyspaceState_value[param.State] + targetState, ok := keyspacepb.KeyspaceState_value[strings.ToUpper(param.State)] if !ok { c.AbortWithStatusJSON(http.StatusBadRequest, errors.Errorf("unknown target state: %s", param.State)) return diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index 560215a01bc..5b1bd2398ac 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -26,7 +26,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/tikv/pd/pkg/testutil" + "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/apiv2/handlers" "github.com/tikv/pd/server/keyspace" "github.com/tikv/pd/tests" @@ -107,29 +107,33 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { keyspaces := mustMakeTestKeyspaces(re, suite.server, 10) for _, created := range keyspaces { // Should NOT allow archiving ENABLED keyspace. - success, _ := sendUpdateStateRequest(re, suite.server, created.Name, "ARCHIVED") + success, _ := sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "archived"}) re.False(success) // Disabling an ENABLED keyspace is allowed. - success, disabled := sendUpdateStateRequest(re, suite.server, created.Name, "DISABLED") + success, disabled := sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "disabled"}) re.True(success) re.Equal(keyspacepb.KeyspaceState_DISABLED, disabled.State) // Disabling an already DISABLED keyspace should not result in any change. - success, disabledAgain := sendUpdateStateRequest(re, suite.server, created.Name, "DISABLED") + success, disabledAgain := sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "disabled"}) re.True(success) re.Equal(disabled, disabledAgain) + // Tombstoning a DISABLED keyspace should not be allowed. + success, _ = sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "tombstone"}) + re.False(success) // Archiving a DISABLED keyspace should be allowed. - success, archived := sendUpdateStateRequest(re, suite.server, created.Name, "ARCHIVED") + success, archived := sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "archived"}) re.True(success) re.Equal(keyspacepb.KeyspaceState_ARCHIVED, archived.State) - // Modifying ARCHIVED keyspace is not allowed. - success, _ = sendUpdateStateRequest(re, suite.server, created.Name, "DISABLED") - re.False(success) - // Using illegal state is not allowed. - success, _ = sendUpdateStateRequest(re, suite.server, created.Name, "UNKNOWN") + // Enabling an ARCHIVED keyspace is not allowed. + success, _ = sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "enabled"}) re.False(success) + // Tombstoning an ARCHIVED keyspace is allowed. + success, tombstone := sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "tombstone"}) + re.True(success) + re.Equal(keyspacepb.KeyspaceState_TOMBSTONE, tombstone.State) } // Changing default keyspace's state is NOT allowed. - success, _ := sendUpdateStateRequest(re, suite.server, keyspace.DefaultKeyspaceName, "DISABLED") + success, _ := sendUpdateStateRequest(re, suite.server, keyspace.DefaultKeyspaceName, &handlers.UpdateStateParam{State: "disabled"}) re.False(success) } @@ -168,19 +172,18 @@ func sendLoadRangeRequest(re *require.Assertions, server *tests.TestServer, toke return resp } -func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, name, newState string) (bool, *keyspacepb.KeyspaceMeta) { - request := handlers.UpdateStateParam{State: newState} +func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, name string, request *handlers.UpdateStateParam) (bool, *keyspacepb.KeyspaceMeta) { data, err := json.Marshal(request) re.NoError(err) httpReq, err := http.NewRequest(http.MethodPut, server.GetAddr()+keyspacesPrefix+"/"+name+"/state", bytes.NewBuffer(data)) re.NoError(err) - resp, err := dialClient.Do(httpReq) + httpResp, err := dialClient.Do(httpReq) re.NoError(err) - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { + defer httpResp.Body.Close() + if httpResp.StatusCode != http.StatusOK { return false, nil } - data, err = io.ReadAll(resp.Body) + data, err = io.ReadAll(httpResp.Body) re.NoError(err) meta := &handlers.KeyspaceMeta{} re.NoError(json.Unmarshal(data, meta)) From fefea0233905af090b641a41aea26cdb5c74984e Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 10 Jan 2023 16:44:05 +0800 Subject: [PATCH 73/76] update kvproto go mod Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- client/go.mod | 5 +---- client/go.sum | 4 ++-- go.mod | 5 +---- go.sum | 9 +++++++-- tests/client/go.mod | 5 +---- tests/client/go.sum | 9 +++++++-- tools/pd-tso-bench/go.sum | 6 ++---- 7 files changed, 21 insertions(+), 22 deletions(-) diff --git a/client/go.mod b/client/go.mod index 97cdab0e858..11514456644 100644 --- a/client/go.mod +++ b/client/go.mod @@ -6,7 +6,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 + github.com/pingcap/kvproto v0.0.0-20230110033234-055843a0a07d github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.11.0 github.com/stretchr/testify v1.7.0 @@ -14,6 +14,3 @@ require ( go.uber.org/zap v1.20.0 google.golang.org/grpc v1.43.0 ) - -// TODO: remove after kvproto merge -replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349 diff --git a/client/go.sum b/client/go.sum index 63e05ac096a..a563556295a 100644 --- a/client/go.sum +++ b/client/go.sum @@ -1,7 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349 h1:aq9/uAnlenI/ensOMpOSvowqzEPQhIoZ9nmadMBCKNs= -github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -106,6 +104,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/kvproto v0.0.0-20230110033234-055843a0a07d h1:BgioocFKd7i9WW3SkLrcqy+3lZrcBCo7ekTcB0GuuMA= +github.com/pingcap/kvproto v0.0.0-20230110033234-055843a0a07d/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/go.mod b/go.mod index 89be2a4f1ab..9975ae89272 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce - github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1 + github.com/pingcap/kvproto v0.0.0-20230110033234-055843a0a07d github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d github.com/pingcap/tidb-dashboard v0.0.0-20221201151320-ea3ee6971f2e @@ -189,6 +189,3 @@ require ( // kvproto at the same time. You can run `go mod tidy` to make it replaced with go-mod style specification. // After the PR to kvproto is merged, remember to comment this out and run `go mod tidy`. // replace github.com/pingcap/kvproto => github.com/$YourPrivateRepo $YourPrivateBranch - -// TODO: remove after kvproto merge -replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349 diff --git a/go.sum b/go.sum index 71070db31ff..6fa00a8db6c 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AlekSi/gocov-xml v1.0.0 h1:4QctJBgXEkbzeKz6PJy6bt3JSPNSN4I2mITYW+eKUoQ= github.com/AlekSi/gocov-xml v1.0.0/go.mod h1:J0qYeZ6tDg4oZubW9mAAgxlqw39PDfoEkzB3HXSbEuA= -github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349 h1:aq9/uAnlenI/ensOMpOSvowqzEPQhIoZ9nmadMBCKNs= -github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= @@ -172,6 +170,7 @@ github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQF github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -186,6 +185,7 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -364,6 +364,9 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMtVcOkjUcuQKh+YrluSo7+7YMCQSzy30= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= +github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/kvproto v0.0.0-20230110033234-055843a0a07d h1:BgioocFKd7i9WW3SkLrcqy+3lZrcBCo7ekTcB0GuuMA= +github.com/pingcap/kvproto v0.0.0-20230110033234-055843a0a07d/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= @@ -689,9 +692,11 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= diff --git a/tests/client/go.mod b/tests/client/go.mod index 11a39fe55de..a41063894e8 100644 --- a/tests/client/go.mod +++ b/tests/client/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1 + github.com/pingcap/kvproto v0.0.0-20230110033234-055843a0a07d github.com/stretchr/testify v1.8.0 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 @@ -166,6 +166,3 @@ replace ( google.golang.org/grpc v1.43.0 => google.golang.org/grpc v1.26.0 google.golang.org/protobuf v1.26.0 => github.com/golang/protobuf v1.3.4 ) - -// TODO: remove after kvproto merge -replace github.com/pingcap/kvproto => github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349 diff --git a/tests/client/go.sum b/tests/client/go.sum index 9453c1feb4c..53b8f095e90 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -1,7 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349 h1:aq9/uAnlenI/ensOMpOSvowqzEPQhIoZ9nmadMBCKNs= -github.com/AmoebaProtozoa/kvproto v0.0.0-20230109085749-856e7a77d349/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= @@ -147,6 +145,7 @@ github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQF github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -160,6 +159,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -326,6 +326,9 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/kvproto v0.0.0-20230110033234-055843a0a07d h1:BgioocFKd7i9WW3SkLrcqy+3lZrcBCo7ekTcB0GuuMA= +github.com/pingcap/kvproto v0.0.0-20230110033234-055843a0a07d/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= @@ -624,9 +627,11 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= diff --git a/tools/pd-tso-bench/go.sum b/tools/pd-tso-bench/go.sum index a7b5f5a2981..6730c4c498e 100644 --- a/tools/pd-tso-bench/go.sum +++ b/tools/pd-tso-bench/go.sum @@ -104,10 +104,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad h1:lGKxsEwdE0pVXzHYD1SQ1vfa3t/bFVU/latrQz8b/w0= -github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 h1:FYgKV9znRQmzVrrJDZ0gUfMIvKLAMU1tu1UKJib8bEQ= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20230110033234-055843a0a07d h1:BgioocFKd7i9WW3SkLrcqy+3lZrcBCo7ekTcB0GuuMA= +github.com/pingcap/kvproto v0.0.0-20230110033234-055843a0a07d/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From 62f7c16261d4cf188e574cb2b276757d6799771a Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 13 Jan 2023 00:37:36 +0800 Subject: [PATCH 74/76] remove unused manager method Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/keyspace/keyspace.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go index e9c4f22e86b..83ac637a9ed 100644 --- a/server/keyspace/keyspace.go +++ b/server/keyspace/keyspace.go @@ -224,26 +224,6 @@ func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, err return meta, err } -// LoadKeyspaceByID returns the keyspace specified by id. -// It returns error if loading or unmarshalling met error or if keyspace does not exist. -func (manager *Manager) LoadKeyspaceByID(id uint32) (*keyspacepb.KeyspaceMeta, error) { - var ( - meta *keyspacepb.KeyspaceMeta - err error - ) - err = manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { - meta, err = manager.store.LoadKeyspaceMeta(txn, id) - if err != nil { - return err - } - if meta == nil { - return ErrKeyspaceNotFound - } - return nil - }) - return meta, err -} - // Mutation represents a single operation to be applied on keyspace config. type Mutation struct { Op OpType From 3ac722b501545d285a86065facec6db710f35b95 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Sun, 29 Jan 2023 13:42:24 +0800 Subject: [PATCH 75/76] address comments Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/apiv2/handlers/keyspace.go | 2 +- tests/server/apiv2/handlers/keyspace_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index ebeb401117c..a047e494a96 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -1,4 +1,4 @@ -// Copyright 2022 TiKV Project Authors. +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index 5b1bd2398ac..e13e13737d7 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 TiKV Project Authors. +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From 186a821564e74544f69659e63754696a5df0fbbe Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 2 Feb 2023 14:03:19 +0800 Subject: [PATCH 76/76] empty commit to trigger ci Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com>