*: supports build pd-server with fail point (#6946)

close #4399

In order can use fail points in testing, need to build a special binary
- supports build pd-server with fail point

Signed-off-by: nolouch <[email protected]>

Makefile

@@ -33,6 +33,10 @@ else
 	BUILD_CGO_ENABLED := 1
 endif
 
+ifeq ($(FAILPOINT), 1)
+	BUILD_TAGS += with_fail
+endif
+
 ifeq ("$(WITH_RACE)", "1")
 	BUILD_FLAGS += -race
 	BUILD_CGO_ENABLED := 1
@@ -73,6 +77,11 @@ PD_SERVER_DEP += dashboard-ui
 pd-server: ${PD_SERVER_DEP}
 	CGO_ENABLED=$(BUILD_CGO_ENABLED) go build $(BUILD_FLAGS) -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -tags "$(BUILD_TAGS)" -o $(BUILD_BIN_PATH)/pd-server cmd/pd-server/main.go
 
+pd-server-failpoint:
+	@$(FAILPOINT_ENABLE)
+	FAILPOINT=1 $(MAKE) pd-server || { $(FAILPOINT_DISABLE); exit 1; }
+	@$(FAILPOINT_DISABLE)
+
 pd-server-basic:
 	SWAGGER=0 DASHBOARD=0 $(MAKE) pd-server
 

pkg/utils/etcdutil/etcdutil.go

@@ -106,6 +106,10 @@ func AddEtcdMember(client *clientv3.Client, urls []string) (*clientv3.MemberAddR
 
 // ListEtcdMembers returns a list of internal etcd members.
 func ListEtcdMembers(client *clientv3.Client) (*clientv3.MemberListResponse, error) {
+	failpoint.Inject("SlowEtcdMemberList", func(val failpoint.Value) {
+		d := val.(int)
+		time.Sleep(time.Duration(d) * time.Second)
+	})
 	ctx, cancel := context.WithTimeout(client.Ctx(), DefaultRequestTimeout)
 	listResp, err := client.MemberList(ctx)
 	cancel()
@@ -132,6 +136,10 @@ func EtcdKVGet(c *clientv3.Client, key string, opts ...clientv3.OpOption) (*clie
 	defer cancel()
 
 	start := time.Now()
+	failpoint.Inject("SlowEtcdKVGet", func(val failpoint.Value) {
+		d := val.(int)
+		time.Sleep(time.Duration(d) * time.Second)
+	})
 	resp, err := clientv3.NewKV(c).Get(ctx, key, opts...)
 	if cost := time.Since(start); cost > DefaultSlowRequestTime {
 		log.Warn("kv gets too slow", zap.String("request-key", key), zap.Duration("cost", cost), errs.ZapError(err))

server/api/failpoint.go

@@ -0,0 +1,22 @@
+// Copyright 2023 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build with_fail
+// +build with_fail
+
+package api
+
+func init() {
+	enableFailPointAPI = true
+}

server/api/router.go

@@ -31,6 +31,9 @@ import (
 	"github.com/unrolled/render"
 )
 
+// enableFailPointAPI enable fail point API handler.
+var enableFailPointAPI bool
+
 // createRouteOption is used to register service for mux.Route
 type createRouteOption func(route *mux.Route)
 
@@ -365,15 +368,13 @@ func createRouter(prefix string, svr *server.Server) *mux.Router {
 	registerFunc(apiRouter, "/admin/reset-ts", tsoAdminHandler.ResetTS, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus))
 
 	// API to set or unset failpoints
-	failpoint.Inject("enableFailpointAPI", func() {
-		// this function will be named to "func2". It may be used in test
+	if enableFailPointAPI {
 		registerPrefix(apiRouter, "/fail", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			// The HTTP handler of failpoint requires the full path to be the failpoint path.
 			r.URL.Path = strings.TrimPrefix(r.URL.Path, prefix+apiPrefix+"/fail")
 			new(failpoint.HttpHandler).ServeHTTP(w, r)
-		}), setAuditBackend("test"))
-	})
-
+		}), setAuditBackend(localLog))
+	}
 	// Deprecated: use /pd/api/v1/health instead.
 	rootRouter.HandleFunc("/health", healthHandler.GetHealthStatus).Methods(http.MethodGet)
 	// Deprecated: use /pd/api/v1/ping instead.