fix(region): allow clean history data (#20040)

cmd/climc/shell/misc/history_data_clean.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package misc
+
+import (
+	"fmt"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	type HistoryDataCleanOptions struct {
+		SERVICE string `help:"Service type"`
+		Day     *int   `default:"30"`
+	}
+	R(&HistoryDataCleanOptions{}, "history-data-clean", "clean history data", func(s *mcclient.ClientSession, args *HistoryDataCleanOptions) error {
+		body, err := modules.HistoryDataClean(s, args.SERVICE, args.Day)
+		if err != nil {
+			return err
+		}
+		fmt.Println(body.PrettyString())
+		return nil
+	})
+}

pkg/cloudcommon/db/history_data_clean.go

@@ -0,0 +1,71 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package db
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/gotypes"
+
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+)
+
+func AddHistoryDataCleanHandler(prefix string, app *appsrv.Application) {
+	prefix = fmt.Sprintf("%s/history-data-clean", prefix)
+	app.AddHandler2("POST", prefix, auth.Authenticate(historyDataCleanHandler), nil, "history_data_clean", nil)
+}
+
+func historyDataCleanHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	userCred := fetchUserCredential(ctx)
+	if !userCred.HasSystemAdminPrivilege() {
+		httperrors.ForbiddenError(ctx, w, "only sysadmin can clean history data")
+		return
+	}
+	input, err := appsrv.FetchJSON(r)
+	if err != nil {
+		httperrors.InputParameterError(ctx, w, "FetchJSON")
+		return
+	}
+	date := time.Now().AddDate(0, -1, 0)
+	if !gotypes.IsNil(input) && input.Contains("day") {
+		day, _ := input.Int("day")
+		date = time.Now().AddDate(0, 0, int(day)*-1)
+	}
+	go func() {
+		for _, manager := range globalTables {
+			if hM, ok := manager.(IHistoryDataManager); ok {
+				start := time.Now()
+				cnt, err := hM.HistoryDataClean(ctx, date)
+				if err != nil {
+					log.Errorf("clean %s data error: %v", manager.Keyword(), err)
+					continue
+				}
+				log.Debugf("clean %d %s history data cost %s", cnt, manager.Keyword(), time.Now().Sub(start).Round(time.Second))
+			}
+		}
+	}()
+	appsrv.SendJSON(w, jsonutils.Marshal(map[string]string{"status": "ok"}))
+}
+
+type IHistoryDataManager interface {
+	HistoryDataClean(ctx context.Context, timeBefor time.Time) (int, error)
+}

pkg/cloudcommon/db/jointbase.go

@@ -16,8 +16,11 @@ package db
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"reflect"
+	"strings"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -27,6 +30,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
@@ -255,3 +259,67 @@ func (model *SJointResourceBase) ValidateUpdateData(
 	}
 	return input, nil
 }
+
+func (manager *SJointResourceBaseManager) HistoryDataClean(ctx context.Context, timeBefor time.Time) (int, error) {
+	q := manager.RawQuery("row_id").IsTrue("deleted").LE("deleted_at", timeBefor)
+	rows, err := q.Rows()
+	if err != nil {
+		if errors.Cause(err) == sql.ErrNoRows {
+			return 0, nil
+		}
+		return 0, errors.Wrap(err, "Query")
+	}
+	defer rows.Close()
+	ids := []string{}
+	for rows.Next() {
+		var id string
+		err := rows.Scan(&id)
+		if err != nil {
+			return 0, errors.Wrap(err, "rows.Scan")
+		}
+		ids = append(ids, id)
+	}
+	var purge = func(ids []string) error {
+		vars := []interface{}{}
+		placeholders := make([]string, len(ids))
+		for i := range placeholders {
+			placeholders[i] = "?"
+			vars = append(vars, ids[i])
+		}
+		placeholder := strings.Join(placeholders, ",")
+		sql := fmt.Sprintf(
+			"delete from %s where row_id in (%s)",
+			manager.TableSpec().Name(), placeholder,
+		)
+		lockman.LockRawObject(ctx, manager.Keyword(), "purge")
+		defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "purge")
+
+		_, err = sqlchemy.GetDB().Exec(
+			sql, vars...,
+		)
+		if err != nil {
+			return errors.Wrapf(err, strings.ReplaceAll(sql, "?", "%s"), vars...)
+		}
+		return nil
+	}
+
+	var splitByLen = func(data []string, splitLen int) [][]string {
+		var result [][]string
+		for i := 0; i < len(data); i += splitLen {
+			end := i + splitLen
+			if end > len(data) {
+				end = len(data)
+			}
+			result = append(result, data[i:end])
+		}
+		return result
+	}
+	idsArr := splitByLen(ids, 100)
+	for i := range idsArr {
+		err = purge(idsArr[i])
+		if err != nil {
+			return 0, err
+		}
+	}
+	return len(ids), nil
+}

pkg/cloudcommon/db/standalone_anon.go

@@ -16,7 +16,10 @@ package db
 
 import (
 	"context"
+	"database/sql"
+	"fmt"
 	"strings"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -27,6 +30,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -1043,3 +1047,67 @@ func GetTagValueCountMap(
 	}
 	return valueMap, nil
 }
+
+func (manager *SStandaloneAnonResourceBaseManager) HistoryDataClean(ctx context.Context, timeBefor time.Time) (int, error) {
+	q := manager.RawQuery("id").IsTrue("deleted").LE("deleted_at", timeBefor)
+	rows, err := q.Rows()
+	if err != nil {
+		if errors.Cause(err) == sql.ErrNoRows {
+			return 0, nil
+		}
+		return 0, errors.Wrap(err, "Query")
+	}
+	defer rows.Close()
+	ids := []string{}
+	for rows.Next() {
+		var id string
+		err := rows.Scan(&id)
+		if err != nil {
+			return 0, errors.Wrap(err, "rows.Scan")
+		}
+		ids = append(ids, id)
+	}
+	var purge = func(ids []string) error {
+		vars := []interface{}{}
+		placeholders := make([]string, len(ids))
+		for i := range placeholders {
+			placeholders[i] = "?"
+			vars = append(vars, ids[i])
+		}
+		placeholder := strings.Join(placeholders, ",")
+		sql := fmt.Sprintf(
+			"delete from %s where id in (%s)",
+			manager.TableSpec().Name(), placeholder,
+		)
+		lockman.LockRawObject(ctx, manager.Keyword(), "purge")
+		defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "purge")
+
+		_, err = sqlchemy.GetDB().Exec(
+			sql, vars...,
+		)
+		if err != nil {
+			return errors.Wrapf(err, strings.ReplaceAll(sql, "?", "%s"), vars...)
+		}
+		return nil
+	}
+
+	var splitByLen = func(data []string, splitLen int) [][]string {
+		var result [][]string
+		for i := 0; i < len(data); i += splitLen {
+			end := i + splitLen
+			if end > len(data) {
+				end = len(data)
+			}
+			result = append(result, data[i:end])
+		}
+		return result
+	}
+	idsArr := splitByLen(ids, 100)
+	for i := range idsArr {
+		err = purge(idsArr[i])
+		if err != nil {
+			return 0, err
+		}
+	}
+	return len(ids), nil
+}

pkg/compute/service/handlers.go

@@ -37,6 +37,7 @@ func InitHandlers(app *appsrv.Application) {
 	db.RegistUserCredCacheUpdater()
 
 	db.AddScopeResourceCountHandler("", app)
+	db.AddHistoryDataCleanHandler("", app)
 
 	quotas.AddQuotaHandler(&models.QuotaManager.SQuotaBaseManager, "", app)
 	quotas.AddQuotaHandler(&models.RegionQuotaManager.SQuotaBaseManager, "", app)

pkg/mcclient/modulebase/historydataclean.go

@@ -0,0 +1,42 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modulebase
+
+import (
+	"io"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+func HistoryDataClean(s *mcclient.ClientSession, serviceType string, day *int) (jsonutils.JSONObject, error) {
+	man := &BaseManager{serviceType: serviceType}
+	input := jsonutils.NewDict()
+	if day != nil {
+		input.Set("day", jsonutils.NewInt(int64(*day)))
+	}
+	resp, err := man.rawRequest(s, "POST", "/history-data-clean", nil, strings.NewReader(input.String()))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	return jsonutils.Parse(body)
+}

pkg/mcclient/modules/historydataclean.go

@@ -0,0 +1,26 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+)
+
+func HistoryDataClean(s *mcclient.ClientSession, serviceType string, day *int) (jsonutils.JSONObject, error) {
+	return modulebase.HistoryDataClean(s, serviceType, day)
+}