Merge pull request #2159 from actiontech/issue-ee-1272-cherry-pick

Issue ee 1272 cherry pick

spelling_dict.txt

@@ -22,6 +22,7 @@ binlog
 blkid
 btree
 chinese
+cidr
 cardinalities
 ccug
 chanxuehong

sqle/api/controller/v1/audit_plan.go

@@ -6,7 +6,9 @@ import (
 	"encoding/csv"
 	"fmt"
 	"mime"
+	"net"
 	"net/http"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -792,21 +794,89 @@ func GetAuditPlanReport(c echo.Context) error {
 }
 
 func filterSQLsByBlackList(sqls []*AuditPlanSQLReqV1, blackList []*model.BlackListAuditPlanSQL) []*AuditPlanSQLReqV1 {
+	if len(blackList) == 0 {
+		return sqls
+	}
 	filteredSQLs := []*AuditPlanSQLReqV1{}
+	filter := ConvertToBlackFilter(blackList)
 	for _, sql := range sqls {
-		var match bool
-		for _, blackSQL := range blackList {
-			// todo: ee issue1119, 临时使用strings.Contains判断子字符串
-			match = strings.Contains(strings.ToUpper(sql.LastReceiveText), strings.ToUpper(blackSQL.FilterSQL))
-			if match {
-				break
+		if filter.IsEndpointInBlackList([]string{sql.Endpoint}) || filter.IsSqlInBlackList(sql.LastReceiveText) {
+			continue
+		}
+		filteredSQLs = append(filteredSQLs, sql)
+	}
+	return filteredSQLs
+}
+
+func ConvertToBlackFilter(blackList []*model.BlackListAuditPlanSQL) *BlackFilter {
+	var blackFilter BlackFilter
+	for _, filter := range blackList {
+		switch filter.FilterType {
+		case model.FilterTypeSQL:
+			blackFilter.BlackSqlList = append(blackFilter.BlackSqlList, utils.FullFuzzySearchRegexp(filter.FilterContent))
+		case model.FilterTypeHost:
+			blackFilter.BlackHostList = append(blackFilter.BlackHostList, utils.FullFuzzySearchRegexp(filter.FilterContent))
+		case model.FilterTypeIP:
+			ip := net.ParseIP(filter.FilterContent)
+			if ip == nil {
+				log.Logger().Errorf("wrong ip in black list,ip:%s", filter.FilterContent)
+				continue
 			}
+			blackFilter.BlackIpList = append(blackFilter.BlackIpList, ip)
+		case model.FilterTypeCIDR:
+			_, cidr, err := net.ParseCIDR(filter.FilterContent)
+			if err != nil {
+				log.Logger().Errorf("wrong cidr in black list,cidr:%s,err:%v", filter.FilterContent, err)
+				continue
+			}
+			blackFilter.BlackCidrList = append(blackFilter.BlackCidrList, cidr)
+		}
+	}
+	return &blackFilter
+}
+
+// 构造BlackFilter的目的是缓存黑名单中需要使用的结构体，在每个循环中复用
+type BlackFilter struct {
+	BlackSqlList  []*regexp.Regexp //更换正则匹配提高效率
+	BlackIpList   []net.IP
+	BlackHostList []*regexp.Regexp
+	BlackCidrList []*net.IPNet
+}
+
+func (f BlackFilter) IsSqlInBlackList(checkSql string) bool {
+	for _, blackSql := range f.BlackSqlList {
+		if blackSql.MatchString(checkSql) {
+			return true
 		}
-		if !match {
-			filteredSQLs = append(filteredSQLs, sql)
+	}
+	return false
+}
+
+func (f BlackFilter) IsEndpointInBlackList(checkIps []string) bool {
+	var checkNetIp net.IP
+	for _, checkIp := range checkIps {
+		checkNetIp = net.ParseIP(checkIp)
+		if checkNetIp == nil {
+			// 无法解析IP，可能是域名，需要正则匹配
+			for _, blackHost := range f.BlackHostList {
+				if blackHost.MatchString(checkIp) {
+					return true
+				}
+			}
+		} else {
+			for _, blackIp := range f.BlackIpList {
+				if blackIp.Equal(checkNetIp) {
+					return true
+				}
+			}
+			for _, blackCidr := range f.BlackCidrList {
+				if blackCidr.Contains(checkNetIp) {
+					return true
+				}
+			}
 		}
 	}
-	return filteredSQLs
+	return false
 }
 
 type FullSyncAuditPlanSQLsReqV1 struct {
@@ -860,7 +930,9 @@ func FullSyncAuditPlanSQLs(c echo.Context) error {
 	} else {
 		l.Warnf("blacklist is not used, err:%v", err)
 	}
-
+	if len(reqSQLs) == 0 {
+		return controller.JSONBaseErrorReq(c, nil)
+	}
 	sqls, err := convertToModelAuditPlanSQL(c, ap, reqSQLs)
 	if err != nil {
 		return controller.JSONBaseErrorReq(c, err)
@@ -907,7 +979,9 @@ func PartialSyncAuditPlanSQLs(c echo.Context) error {
 	} else {
 		l.Warnf("blacklist is not used, err:%v", err)
 	}
-
+	if len(reqSQLs) == 0 {
+		return controller.JSONBaseErrorReq(c, nil)
+	}
 	sqls, err := convertToModelAuditPlanSQL(c, ap, reqSQLs)
 	if err != nil {
 		return controller.JSONBaseErrorReq(c, err)

sqle/api/controller/v1/audit_plan_test.go

@@ -0,0 +1,135 @@
+package v1_test
+
+import (
+	"testing"
+
+	v1 "github.com/actiontech/sqle/sqle/api/controller/v1"
+	"github.com/actiontech/sqle/sqle/model"
+)
+
+func TestIsSqlInBlackList(t *testing.T) {
+	filter := v1.ConvertToBlackFilter([]*model.BlackListAuditPlanSQL{
+		{
+			FilterContent: "SELECT",
+			FilterType:    "SQL",
+		}, {
+			FilterContent: "table_1",
+			FilterType:    "SQL",
+		},
+	})
+
+	matchSqls := []string{
+		"SELECT * FROM users",
+		"DELETE From tAble_1",
+		"SELECT COUNT(*) FROM table_2",
+	}
+	for _, matchSql := range matchSqls {
+		if !filter.IsSqlInBlackList(matchSql) {
+			t.Error("Expected SQL to match blacklist")
+		}
+	}
+	notMatchSqls := []string{
+		"INSERT INTO users VALUES (1, 'John')",
+		"DELETE  From schools",
+		"SHOW CREATE TABLE table_2",
+	}
+	for _, notMatchSql := range notMatchSqls {
+		if filter.IsSqlInBlackList(notMatchSql) {
+			t.Error("Did not expect SQL to match blacklist")
+		}
+	}
+}
+
+func TestIsIpInBlackList(t *testing.T) {
+	filter := v1.ConvertToBlackFilter([]*model.BlackListAuditPlanSQL{
+		{
+			FilterContent: "192.168.1.23",
+			FilterType:    "IP",
+		}, {
+			FilterContent: "10.0.5.67",
+			FilterType:    "IP",
+		},
+	})
+
+	matchIps := []string{
+		"10.0.5.67",
+		"192.168.1.23",
+	}
+
+	if !filter.IsEndpointInBlackList(matchIps) {
+		t.Error("Expected Ip to match blacklist")
+	}
+
+	notMatchIps := []string{
+		"172.16.254.89",
+		"134.12.45.78",
+		"50.67.89.12",
+	}
+	if filter.IsEndpointInBlackList(notMatchIps) {
+		t.Error("Did not expect Ip to match blacklist")
+	}
+}
+
+func TestIsCidrInBlackList(t *testing.T) {
+	filter := v1.ConvertToBlackFilter([]*model.BlackListAuditPlanSQL{
+		{
+			FilterContent: "192.168.0.0/24",
+			FilterType:    "CIDR",
+		}, {
+			FilterContent: "10.100.0.0/16",
+			FilterType:    "CIDR",
+		},
+	})
+
+	matchIps := []string{
+		"10.100.1.2",
+		"10.100.25.45",
+		"172.30.1.2",
+		"172.30.30.45",
+	}
+
+	if !filter.IsEndpointInBlackList(matchIps) {
+		t.Error("Expected CIDR to match blacklist")
+	}
+
+	notMatchIps := []string{
+		"172.16.254.89",
+		"134.12.45.78",
+		"50.67.89.12",
+	}
+	if filter.IsEndpointInBlackList(notMatchIps) {
+		t.Error("Did not expect CIDR to match blacklist")
+	}
+}
+
+func TestIsHostInBlackList(t *testing.T) {
+	filter := v1.ConvertToBlackFilter([]*model.BlackListAuditPlanSQL{
+		{
+			FilterContent: "test",
+			FilterType:    "HOST",
+		}, {
+			FilterContent: "some_site",
+			FilterType:    "HOST",
+		},
+	})
+
+	matchHosts := []string{
+		"localtest",
+		"localtest.com",
+		"anyTest.io",
+		"some-Site.org/home/",
+		"Some_site.cn/mysql",
+	}
+
+	if !filter.IsEndpointInBlackList(matchHosts) {
+		t.Error("Expected HOST to match blacklist")
+	}
+
+	notMatchHosts := []string{
+		"other_site/home",
+		"any_other_site/local",
+	}
+	if filter.IsEndpointInBlackList(notMatchHosts) {
+		t.Error("Did not expect HOST to match blacklist")
+	}
+}

sqle/api/controller/v2/audit_plan.go

@@ -279,19 +279,16 @@ type AuditPlanSQLReqV2 struct {
 }
 
 func filterSQLsByBlackList(sqls []*AuditPlanSQLReqV2, blackList []*model.BlackListAuditPlanSQL) []*AuditPlanSQLReqV2 {
+	if len(blackList) == 0 {
+		return sqls
+	}
 	filteredSQLs := []*AuditPlanSQLReqV2{}
+	filter := v1.ConvertToBlackFilter(blackList)
 	for _, sql := range sqls {
-		var match bool
-		for _, blackSQL := range blackList {
-			// todo: ee issue1119, 临时使用strings.Contains判断子字符串
-			match = strings.Contains(strings.ToUpper(sql.LastReceiveText), strings.ToUpper(blackSQL.FilterSQL))
-			if match {
-				break
-			}
-		}
-		if !match {
-			filteredSQLs = append(filteredSQLs, sql)
+		if filter.IsEndpointInBlackList(sql.Endpoints) || filter.IsSqlInBlackList(sql.LastReceiveText) {
+			continue
 		}
+		filteredSQLs = append(filteredSQLs, sql)
 	}
 	return filteredSQLs
 }
@@ -413,7 +410,9 @@ func PartialSyncAuditPlanSQLs(c echo.Context) error {
 	} else {
 		l.Warnf("blacklist is not used, err:%v", err)
 	}
-
+	if len(reqSQLs) == 0 {
+		return controller.JSONBaseErrorReq(c, nil)
+	}
 	sqls, err := convertToModelAuditPlanSQL(c, ap, reqSQLs)
 	if err != nil {
 		return controller.JSONBaseErrorReq(c, err)
@@ -457,7 +456,9 @@ func FullSyncAuditPlanSQLs(c echo.Context) error {
 	} else {
 		l.Warnf("blacklist is not used, err:%v", err)
 	}
-
+	if len(reqSQLs) == 0 {
+		return controller.JSONBaseErrorReq(c, nil)
+	}
 	sqls, err := convertToModelAuditPlanSQL(c, ap, reqSQLs)
 	if err != nil {
 		return controller.JSONBaseErrorReq(c, err)

sqle/model/audit_plan.go

@@ -52,10 +52,21 @@ type AuditPlanSQLV2 struct {
 	Schema         string `json:"schema" gorm:"type:varchar(512);not null"`
 }
 
+const (
+	FilterTypeSQL  string = "SQL"
+	FilterTypeIP   string = "IP"
+	FilterTypeCIDR string = "CIDR"
+	FilterTypeHost string = "HOST"
+)
+
 type BlackListAuditPlanSQL struct {
 	Model
+	FilterContent string `json:"filter_content" gorm:"type:varchar(512);not null;"`
+	FilterType    string `json:"filter_type" gorm:"type:enum('SQL','IP','CIDR','HOST');default:'SQL';not null;"`
+}
 
-	FilterSQL string `json:"filter_sql" gorm:"type:varchar(512);not null;unique"`
+func (a BlackListAuditPlanSQL) TableName() string {
+	return "black_list_audit_plan_sqls"
 }
 
 func (s *Storage) GetBlackListAuditPlanSQLs() ([]*BlackListAuditPlanSQL, error) {

sqle/model/utils.go

@@ -190,6 +190,10 @@ func (s *Storage) AutoMigrate() error {
 		return errors.New(errors.ConnectStorageError, err)
 	}
 
+	err = s.db.Model(BlackListAuditPlanSQL{}).AddUniqueIndex("uniq_type_content", "filter_type", "filter_content").Error
+	if err != nil {
+		return errors.New(errors.ConnectStorageError, err)
+	}
 	if s.db.Dialect().HasColumn(Rule{}.TableName(), "is_default") {
 		if err = s.db.Model(&Rule{}).DropColumn("is_default").Error; err != nil {
 			return errors.New(errors.ConnectStorageError, err)

sqle/utils/util.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"math"
 	"net/url"
+	"regexp"
 	"strconv"
 	"strings"
 	"sync"
@@ -308,3 +309,8 @@ func IsPrefixSubStrArray(arr []string, prefix []string) bool {
 
 	return true
 }
+
+// 全模糊匹配字符串，并且对大小写不敏感
+func FullFuzzySearchRegexp(str string) *regexp.Regexp {
+	return regexp.MustCompile(`^.*(?i)` + regexp.QuoteMeta(str) + `.*$`)
+}