From 9a2853a9013136c298d0cdb7443a109cbc795d04 Mon Sep 17 00:00:00 2001 From: winfredLIN Date: Thu, 14 Dec 2023 15:08:36 +0800 Subject: [PATCH 1/9] feat: add three types of slow log blacklist: IP, HOST, and CIDR, as well as filtering methods --- sqle/api/controller/v1/audit_plan.go | 96 ++++++++++++++++++++++++---- sqle/api/controller/v2/audit_plan.go | 25 ++++---- sqle/model/audit_plan.go | 13 +++- sqle/model/utils.go | 4 ++ sqle/utils/util.go | 5 ++ 5 files changed, 119 insertions(+), 24 deletions(-) diff --git a/sqle/api/controller/v1/audit_plan.go b/sqle/api/controller/v1/audit_plan.go index 54a4cfc55e..f2783ddfdf 100644 --- a/sqle/api/controller/v1/audit_plan.go +++ b/sqle/api/controller/v1/audit_plan.go @@ -6,7 +6,9 @@ import ( "encoding/csv" "fmt" "mime" + "net" "net/http" + "regexp" "strconv" "strings" "time" @@ -792,21 +794,89 @@ func GetAuditPlanReport(c echo.Context) error { } func filterSQLsByBlackList(sqls []*AuditPlanSQLReqV1, blackList []*model.BlackListAuditPlanSQL) []*AuditPlanSQLReqV1 { + if len(blackList) == 0 { + return sqls + } filteredSQLs := []*AuditPlanSQLReqV1{} + filter := ConvertToBlackFilter(blackList) for _, sql := range sqls { - var match bool - for _, blackSQL := range blackList { - // todo: ee issue1119, 临时使用strings.Contains判断子字符串 - match = strings.Contains(strings.ToUpper(sql.LastReceiveText), strings.ToUpper(blackSQL.FilterSQL)) - if match { - break + if filter.IsIpInBlackList([]string{sql.Endpoint}) || filter.IsSqlInBlackList(sql.LastReceiveText) { + continue + } + filteredSQLs = append(filteredSQLs, sql) + } + return filteredSQLs +} + +func ConvertToBlackFilter(blackList []*model.BlackListAuditPlanSQL) *BlackFilter { + var blackFilter BlackFilter + for _, filter := range blackList { + switch filter.FilterType { + case model.FilterTypeSQL: + blackFilter.BlackSqlList = append(blackFilter.BlackSqlList, utils.FullFuzzySearchRegexp(filter.FilterContent)) + case model.FilterTypeHost: + blackFilter.BlackHostList = append(blackFilter.BlackHostList, utils.FullFuzzySearchRegexp(filter.FilterContent)) + case model.FilterTypeIP: + ip := net.ParseIP(filter.FilterContent) + if ip == nil { + log.Logger().Errorf("wrong ip in black list,ip:%s", filter.FilterContent) + continue } + blackFilter.BlackIpList = append(blackFilter.BlackIpList, ip) + case model.FilterTypeCIDR: + _, cidr, err := net.ParseCIDR(filter.FilterContent) + if err != nil { + log.Logger().Errorf("wrong cidr in black list,cidr:%s,err:%v", filter.FilterContent, err) + continue + } + blackFilter.BlackCidrList = append(blackFilter.BlackCidrList, cidr) + } + } + return &blackFilter +} + +// 构造BlackFilter的目的是缓存黑名单中需要使用的结构体，在每个循环中复用 +type BlackFilter struct { + BlackSqlList []*regexp.Regexp //更换正则匹配提高效率 + BlackIpList []net.IP + BlackHostList []*regexp.Regexp + BlackCidrList []*net.IPNet +} + +func (f BlackFilter) IsSqlInBlackList(checkSql string) bool { + for _, blackSql := range f.BlackSqlList { + if blackSql.MatchString(checkSql) { + return true } - if !match { - filteredSQLs = append(filteredSQLs, sql) + } + return false +} + +func (f BlackFilter) IsIpInBlackList(checkIps []string) bool { + var checkNetIp net.IP + for _, checkIp := range checkIps { + checkNetIp = net.ParseIP(checkIp) + if checkNetIp == nil { + // 无法解析IP，可能是域名，需要正则匹配 + for _, BlackHost := range f.BlackHostList { + if BlackHost.MatchString(checkIp) { + return true + } + } + } else { + for _, blackIp := range f.BlackIpList { + if blackIp.Equal(checkNetIp) { + return true + } + } + for _, blackCidr := range f.BlackCidrList { + if blackCidr.Contains(checkNetIp) { + return true + } + } } } - return filteredSQLs + return false } type FullSyncAuditPlanSQLsReqV1 struct { @@ -860,7 +930,9 @@ func FullSyncAuditPlanSQLs(c echo.Context) error { } else { l.Warnf("blacklist is not used, err:%v", err) } - + if len(reqSQLs) == 0 { + return controller.JSONBaseErrorReq(c, nil) + } sqls, err := convertToModelAuditPlanSQL(c, ap, reqSQLs) if err != nil { return controller.JSONBaseErrorReq(c, err) @@ -907,7 +979,9 @@ func PartialSyncAuditPlanSQLs(c echo.Context) error { } else { l.Warnf("blacklist is not used, err:%v", err) } - + if len(reqSQLs) == 0 { + return controller.JSONBaseErrorReq(c, nil) + } sqls, err := convertToModelAuditPlanSQL(c, ap, reqSQLs) if err != nil { return controller.JSONBaseErrorReq(c, err) diff --git a/sqle/api/controller/v2/audit_plan.go b/sqle/api/controller/v2/audit_plan.go index 0825beae19..e5f34ddf9d 100644 --- a/sqle/api/controller/v2/audit_plan.go +++ b/sqle/api/controller/v2/audit_plan.go @@ -279,19 +279,16 @@ type AuditPlanSQLReqV2 struct { } func filterSQLsByBlackList(sqls []*AuditPlanSQLReqV2, blackList []*model.BlackListAuditPlanSQL) []*AuditPlanSQLReqV2 { + if len(blackList) == 0 { + return sqls + } filteredSQLs := []*AuditPlanSQLReqV2{} + filter := v1.ConvertToBlackFilter(blackList) for _, sql := range sqls { - var match bool - for _, blackSQL := range blackList { - // todo: ee issue1119, 临时使用strings.Contains判断子字符串 - match = strings.Contains(strings.ToUpper(sql.LastReceiveText), strings.ToUpper(blackSQL.FilterSQL)) - if match { - break - } - } - if !match { - filteredSQLs = append(filteredSQLs, sql) + if filter.IsIpInBlackList(sql.Endpoints) || filter.IsSqlInBlackList(sql.LastReceiveText) { + continue } + filteredSQLs = append(filteredSQLs, sql) } return filteredSQLs } @@ -413,7 +410,9 @@ func PartialSyncAuditPlanSQLs(c echo.Context) error { } else { l.Warnf("blacklist is not used, err:%v", err) } - + if len(reqSQLs) == 0 { + return controller.JSONBaseErrorReq(c, nil) + } sqls, err := convertToModelAuditPlanSQL(c, ap, reqSQLs) if err != nil { return controller.JSONBaseErrorReq(c, err) @@ -457,7 +456,9 @@ func FullSyncAuditPlanSQLs(c echo.Context) error { } else { l.Warnf("blacklist is not used, err:%v", err) } - + if len(reqSQLs) == 0 { + return controller.JSONBaseErrorReq(c, nil) + } sqls, err := convertToModelAuditPlanSQL(c, ap, reqSQLs) if err != nil { return controller.JSONBaseErrorReq(c, err) diff --git a/sqle/model/audit_plan.go b/sqle/model/audit_plan.go index e303e76b79..31095bb2fd 100644 --- a/sqle/model/audit_plan.go +++ b/sqle/model/audit_plan.go @@ -52,10 +52,21 @@ type AuditPlanSQLV2 struct { Schema string `json:"schema" gorm:"type:varchar(512);not null"` } +const ( + FilterTypeSQL string = "SQL" + FilterTypeIP string = "IP" + FilterTypeCIDR string = "CIDR" + FilterTypeHost string = "HOST" +) + type BlackListAuditPlanSQL struct { Model + FilterContent string `json:"filter_content" gorm:"type:varchar(512);not null;"` + FilterType string `json:"filter_type" gorm:"type:enum('SQL','IP','CIDR','HOST');default:'SQL';not null;"` +} - FilterSQL string `json:"filter_sql" gorm:"type:varchar(512);not null;unique"` +func (a BlackListAuditPlanSQL) TableName() string { + return "black_list_audit_plan_sqls" } func (s *Storage) GetBlackListAuditPlanSQLs() ([]*BlackListAuditPlanSQL, error) { diff --git a/sqle/model/utils.go b/sqle/model/utils.go index c58480b9ea..e393353b11 100644 --- a/sqle/model/utils.go +++ b/sqle/model/utils.go @@ -190,6 +190,10 @@ func (s *Storage) AutoMigrate() error { return errors.New(errors.ConnectStorageError, err) } + err = s.db.Model(BlackListAuditPlanSQL{}).AddUniqueIndex("uniq_type_content", "filter_type", "filter_content").Error + if err != nil { + return errors.New(errors.ConnectStorageError, err) + } if s.db.Dialect().HasColumn(Rule{}.TableName(), "is_default") { if err = s.db.Model(&Rule{}).DropColumn("is_default").Error; err != nil { return errors.New(errors.ConnectStorageError, err) diff --git a/sqle/utils/util.go b/sqle/utils/util.go index 6f8fa4df0b..72872884c3 100644 --- a/sqle/utils/util.go +++ b/sqle/utils/util.go @@ -7,6 +7,7 @@ import ( "fmt" "math" "net/url" + "regexp" "strconv" "strings" "sync" @@ -308,3 +309,7 @@ func IsPrefixSubStrArray(arr []string, prefix []string) bool { return true } + +func FullFuzzySearchRegexp(str string) *regexp.Regexp { + return regexp.MustCompile(`^.*(?i)` + str + `.*$`) +} From acd2b5d2bac60bcab1788be12494c62884f6bb5d Mon Sep 17 00:00:00 2001 From: winfredLIN Date: Thu, 14 Dec 2023 15:09:17 +0800 Subject: [PATCH 2/9] test: add unit test for function FullFuzzySearchRegexp --- sqle/utils/util_test.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/sqle/utils/util_test.go b/sqle/utils/util_test.go index 6492f2ed14..250913f1fe 100644 --- a/sqle/utils/util_test.go +++ b/sqle/utils/util_test.go @@ -239,3 +239,40 @@ func TestIsGitHttpURL(t *testing.T) { assert.False(t, IsGitHttpURL(tc), "Expected %q to be an invalid Git Http URL", tc) } } + +func TestFullFuzzySearchRegexp(t *testing.T) { + testCases := []struct { + input string + wantMatch []string + wantNoMatch []string + }{ + { + "Hello", + []string{"heyHelloCode", "HElLO", "Sun_hello", "HelLo_Jack"}, + []string{"GoLang is awesome", "I love GOLANG", "GoLangGOLANGGolang"}, + }, + { + "Golang", + []string{"GoLang is awesome", "I love GOLANG", "GoLangGOLANGGolang"}, + []string{"language", "hi", "heyHelloCode", "HElLO", "Sun_hello", "HelLo_Jack"}, + }, + } + + for _, tc := range testCases { + reg := FullFuzzySearchRegexp(tc.input) + + // Positive cases + for _, s := range tc.wantMatch { + if !reg.MatchString(s) { + t.Errorf("Expected %q to match %v", s, reg) + } + } + + // Negative cases + for _, s := range tc.wantNoMatch { + if reg.MatchString(s) { + t.Errorf("Expected %q NOT to match %v", s, reg) + } + } + } +} From 86d16b3770111228d4b53c729775a7d49a83acd3 Mon Sep 17 00:00:00 2001 From: winfredLIN Date: Thu, 14 Dec 2023 15:09:51 +0800 Subject: [PATCH 3/9] ci: add word cidr into white list --- spelling_dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/spelling_dict.txt b/spelling_dict.txt index b0ebb89931..46a6d4c646 100644 --- a/spelling_dict.txt +++ b/spelling_dict.txt @@ -22,6 +22,7 @@ binlog blkid btree chinese +cidr cardinalities ccug chanxuehong From 44b598dfa324c03e6d5fbd50e1f53c5cb743c833 Mon Sep 17 00:00:00 2001 From: winfredLIN Date: Fri, 15 Dec 2023 15:39:37 +0800 Subject: [PATCH 4/9] style: modify name of function --- sqle/api/controller/v1/audit_plan.go | 8 ++++---- sqle/api/controller/v2/audit_plan.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sqle/api/controller/v1/audit_plan.go b/sqle/api/controller/v1/audit_plan.go index f2783ddfdf..4212f4cb4b 100644 --- a/sqle/api/controller/v1/audit_plan.go +++ b/sqle/api/controller/v1/audit_plan.go @@ -800,7 +800,7 @@ func filterSQLsByBlackList(sqls []*AuditPlanSQLReqV1, blackList []*model.BlackLi filteredSQLs := []*AuditPlanSQLReqV1{} filter := ConvertToBlackFilter(blackList) for _, sql := range sqls { - if filter.IsIpInBlackList([]string{sql.Endpoint}) || filter.IsSqlInBlackList(sql.LastReceiveText) { + if filter.IsEndpointInBlackList([]string{sql.Endpoint}) || filter.IsSqlInBlackList(sql.LastReceiveText) { continue } filteredSQLs = append(filteredSQLs, sql) @@ -852,14 +852,14 @@ func (f BlackFilter) IsSqlInBlackList(checkSql string) bool { return false } -func (f BlackFilter) IsIpInBlackList(checkIps []string) bool { +func (f BlackFilter) IsEndpointInBlackList(checkIps []string) bool { var checkNetIp net.IP for _, checkIp := range checkIps { checkNetIp = net.ParseIP(checkIp) if checkNetIp == nil { // 无法解析IP，可能是域名，需要正则匹配 - for _, BlackHost := range f.BlackHostList { - if BlackHost.MatchString(checkIp) { + for _, blackHost := range f.BlackHostList { + if blackHost.MatchString(checkIp) { return true } } diff --git a/sqle/api/controller/v2/audit_plan.go b/sqle/api/controller/v2/audit_plan.go index e5f34ddf9d..2571d3ede0 100644 --- a/sqle/api/controller/v2/audit_plan.go +++ b/sqle/api/controller/v2/audit_plan.go @@ -285,7 +285,7 @@ func filterSQLsByBlackList(sqls []*AuditPlanSQLReqV2, blackList []*model.BlackLi filteredSQLs := []*AuditPlanSQLReqV2{} filter := v1.ConvertToBlackFilter(blackList) for _, sql := range sqls { - if filter.IsIpInBlackList(sql.Endpoints) || filter.IsSqlInBlackList(sql.LastReceiveText) { + if filter.IsEndpointInBlackList(sql.Endpoints) || filter.IsSqlInBlackList(sql.LastReceiveText) { continue } filteredSQLs = append(filteredSQLs, sql) From 137e81685e10c047c8e99e3414c49469d81aae6c Mon Sep 17 00:00:00 2001 From: winfredLIN Date: Fri, 15 Dec 2023 16:05:52 +0800 Subject: [PATCH 5/9] modify: prevent injection of regularization --- sqle/utils/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqle/utils/util.go b/sqle/utils/util.go index 72872884c3..009fa2836a 100644 --- a/sqle/utils/util.go +++ b/sqle/utils/util.go @@ -311,5 +311,5 @@ func IsPrefixSubStrArray(arr []string, prefix []string) bool { } func FullFuzzySearchRegexp(str string) *regexp.Regexp { - return regexp.MustCompile(`^.*(?i)` + str + `.*$`) + return regexp.MustCompile(`^.*(?i)` + regexp.QuoteMeta(str) + `.*$`) } From 93b1c5fec40e4e936a8501336ea135e31a8011e5 Mon Sep 17 00:00:00 2001 From: winfredLIN Date: Fri, 15 Dec 2023 16:06:04 +0800 Subject: [PATCH 6/9] test: add test case for preventing injection of regularization --- sqle/utils/util_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sqle/utils/util_test.go b/sqle/utils/util_test.go index 250913f1fe..4fb40319e7 100644 --- a/sqle/utils/util_test.go +++ b/sqle/utils/util_test.go @@ -255,6 +255,10 @@ func TestFullFuzzySearchRegexp(t *testing.T) { "Golang", []string{"GoLang is awesome", "I love GOLANG", "GoLangGOLANGGolang"}, []string{"language", "hi", "heyHelloCode", "HElLO", "Sun_hello", "HelLo_Jack"}, + }, { + ".*(?i)", + []string{"GoLang .*(?i) awesome", "I love GO^.*(?i)SING", "GoLangGO.*(?i)Golang"}, + []string{"language", "hi", "heyHelloCode", "HElLO", "Sun_hello", "HelLo_Jack"}, }, } From 3052b897cef84803e47b09ff4065a4cd3a445739 Mon Sep 17 00:00:00 2001 From: winfredLIN Date: Fri, 15 Dec 2023 16:58:41 +0800 Subject: [PATCH 7/9] test: add test for black list --- sqle/api/controller/v1/audit_plan_test.go | 134 ++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 sqle/api/controller/v1/audit_plan_test.go diff --git a/sqle/api/controller/v1/audit_plan_test.go b/sqle/api/controller/v1/audit_plan_test.go new file mode 100644 index 0000000000..331624933c --- /dev/null +++ b/sqle/api/controller/v1/audit_plan_test.go @@ -0,0 +1,134 @@ +package v1_test + +import ( + "testing" + + v1 "github.com/actiontech/sqle/sqle/api/controller/v1" + "github.com/actiontech/sqle/sqle/model" +) + +func TestIsSqlInBlackList(t *testing.T) { + filter := v1.ConvertToBlackFilter([]*model.BlackListAuditPlanSQL{ + { + FilterContent: "SELECT", + FilterType: "SQL", + }, { + FilterContent: "table_1", + FilterType: "SQL", + }, + }) + + matchSqls := []string{ + "SELECT * FROM users", + "DELETE From tAble_1", + "SELECT COUNT(*) FROM table_2", + } + for _, matchSql := range matchSqls { + if !filter.IsSqlInBlackList(matchSql) { + t.Error("Expected SQL to match blacklist") + } + } + notMatchSqls := []string{ + "INSERT INTO users VALUES (1, 'John')", + "DELETE From schools", + "SHOW CREATE TABLE table_2", + } + for _, notMatchSql := range notMatchSqls { + if filter.IsSqlInBlackList(notMatchSql) { + t.Error("Did not expect SQL to match blacklist") + } + } +} + +func TestIsIpInBlackList(t *testing.T) { + filter := v1.ConvertToBlackFilter([]*model.BlackListAuditPlanSQL{ + { + FilterContent: "192.168.1.23", + FilterType: "IP", + }, { + FilterContent: "10.0.5.67", + FilterType: "IP", + }, + }) + + matchIps := []string{ + "10.0.5.67", + "192.168.1.23", + } + + if !filter.IsEndpointInBlackList(matchIps) { + t.Error("Expected Ip to match blacklist") + } + + notMatchIps := []string{ + "172.16.254.89", + "134.12.45.78", + "50.67.89.12", + } + if filter.IsEndpointInBlackList(notMatchIps) { + t.Error("Did not expect Ip to match blacklist") + } +} + +func TestIsCidrInBlackList(t *testing.T) { + filter := v1.ConvertToBlackFilter([]*model.BlackListAuditPlanSQL{ + { + FilterContent: "192.168.0.0/24", + FilterType: "CIDR", + }, { + FilterContent: "10.100.0.0/16", + FilterType: "CIDR", + }, + }) + + matchIps := []string{ + "10.100.1.2", + "10.100.25.45", + "172.30.1.2", + "172.30.30.45", + } + + if !filter.IsEndpointInBlackList(matchIps) { + t.Error("Expected CIDR to match blacklist") + } + + notMatchIps := []string{ + "172.16.254.89", + "134.12.45.78", + "50.67.89.12", + } + if filter.IsEndpointInBlackList(notMatchIps) { + t.Error("Did not expect CIDR to match blacklist") + } +} + +func TestIsHostInBlackList(t *testing.T) { + filter := v1.ConvertToBlackFilter([]*model.BlackListAuditPlanSQL{ + { + FilterContent: "test", + FilterType: "HOST", + }, { + FilterContent: "some_site", + FilterType: "HOST", + }, + }) + + matchHosts := []string{ + "localtest", + "anytest", + "some_Site/home/", + "Some_site/mysql", + } + + if !filter.IsEndpointInBlackList(matchHosts) { + t.Error("Expected HOST to match blacklist") + } + + notMatchHosts := []string{ + "other_site/home", + "any_other_site/local", + } + if filter.IsEndpointInBlackList(notMatchHosts) { + t.Error("Did not expect HOST to match blacklist") + } +} From 86cbca1efa6bf74d80f3b4531a0faf7370e9f8d3 Mon Sep 17 00:00:00 2001 From: winfredLIN Date: Fri, 15 Dec 2023 17:00:35 +0800 Subject: [PATCH 8/9] style: add comment --- sqle/utils/util.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sqle/utils/util.go b/sqle/utils/util.go index 009fa2836a..d4ac810993 100644 --- a/sqle/utils/util.go +++ b/sqle/utils/util.go @@ -310,6 +310,7 @@ func IsPrefixSubStrArray(arr []string, prefix []string) bool { return true } +// 全模糊匹配字符串，并且对大小写不敏感 func FullFuzzySearchRegexp(str string) *regexp.Regexp { return regexp.MustCompile(`^.*(?i)` + regexp.QuoteMeta(str) + `.*$`) } From b44beb155def8905bf1fdfae4e062b1e7dc7dc0b Mon Sep 17 00:00:00 2001 From: winfredLIN Date: Fri, 15 Dec 2023 17:17:29 +0800 Subject: [PATCH 9/9] test: modify test cases --- sqle/api/controller/v1/audit_plan_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sqle/api/controller/v1/audit_plan_test.go b/sqle/api/controller/v1/audit_plan_test.go index 331624933c..9f90e2304f 100644 --- a/sqle/api/controller/v1/audit_plan_test.go +++ b/sqle/api/controller/v1/audit_plan_test.go @@ -115,9 +115,10 @@ func TestIsHostInBlackList(t *testing.T) { matchHosts := []string{ "localtest", - "anytest", - "some_Site/home/", - "Some_site/mysql", + "localtest.com", + "anyTest.io", + "some-Site.org/home/", + "Some_site.cn/mysql", } if !filter.IsEndpointInBlackList(matchHosts) {