Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

support directly audit several files #1751

Merged
merged 4 commits into from
Aug 24, 2023
Merged

support directly audit several files #1751

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
de73587
support directly audit several files
ColdWaterLW Aug 24, 2023
4bd03f1
alert when parse SQL failed
ColdWaterLW Aug 24, 2023
ffa6569
check privilege
ColdWaterLW Aug 24, 2023
0460a7b
fix spelling
ColdWaterLW Aug 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/99designs/gqlgen v0.17.20
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/Masterminds/semver/v3 v3.1.1
github.com/actiontech/mybatis-mapper-2-sql v0.3.0
github.com/actiontech/mybatis-mapper-2-sql v0.4.0
github.com/agiledragon/gomonkey v2.0.2+incompatible
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
github.com/alibabacloud-go/darabonba-openapi v0.1.18
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,8 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrU
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/VividCortex/mysqlerr v0.0.0-20201215173831-4c396ae82aac/go.mod h1:f3HiCrHjHBdcm6E83vGaXh1KomZMA2P6aeo3hKx/wg0=
github.com/actiontech/mybatis-mapper-2-sql v0.3.0 h1:EMVKKgnCaqj1+3p21yfTIZpXYUUwv7tnKGvDQKRTZok=
github.com/actiontech/mybatis-mapper-2-sql v0.3.0/go.mod h1:C2CICYfJ/MRX9XpBqjp15MXN3X15mlhDJc35CTZTzWQ=
github.com/actiontech/mybatis-mapper-2-sql v0.4.0 h1:FSzK3qnnD9/JjOUJMRcctGxQjK7cg1M01yRMq59TSZg=
github.com/actiontech/mybatis-mapper-2-sql v0.4.0/go.mod h1:ZMmUEDfbjm8oWxSAZkejqeOzlXa1BWNCfhNIxCMu7lw=
github.com/agiledragon/gomonkey v2.0.2+incompatible h1:eXKi9/piiC3cjJD1658mEE2o3NjkJ5vDLgYjCQu0Xlw=
github.com/agiledragon/gomonkey v2.0.2+incompatible/go.mod h1:2NGfXu1a80LLr2cmWXGBDaHEjb1idR6+FVlX5T3D9hw=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
Expand Down
2 changes: 1 addition & 1 deletion sqle/api/app.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -366,7 +366,7 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config config.SqleConfi
// sql audit
v1Router.POST("/sql_audit", v1.DirectAudit)
v2Router.POST("/sql_audit", v2.DirectAudit)

v1Router.POST("/audit_files", v1.DirectAuditFiles)
v1Router.GET("/sql_analysis", v1.DirectGetSQLAnalysis)

// UI
Expand Down
99 changes: 99 additions & 0 deletions sqle/api/controller/v1/sql_audit.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package v1

import (
e "errors"
"fmt"
"net/http"
"strings"

parser "github.com/actiontech/mybatis-mapper-2-sql"
"github.com/actiontech/sqle/sqle/api/controller"
Expand Down Expand Up @@ -50,6 +52,7 @@ type AuditSQLResV1 struct {

var ErrDirectAudit = errors.New(errors.GenericError, fmt.Errorf("audit failed, please confirm whether the type of audit plugin supports static audit, please check the log for details"))

// @Deprecated
// @Summary 直接审核SQL
// @Description Direct audit sql
// @Id directAuditV1
Expand All @@ -65,6 +68,16 @@ func DirectAudit(c echo.Context) error {
return err
}

if req.ProjectName != nil {
user := controller.GetUserName(c)
s := model.GetStorage()
if yes, err := s.IsProjectMember(user, *req.ProjectName); err != nil {
return controller.JSONBaseErrorReq(c, fmt.Errorf("check privilege failed: %v", err))
} else if !yes {
return controller.JSONBaseErrorReq(c, errors.New(errors.ErrAccessDeniedError, e.New("you are not the project member")))
}
}

sql := req.SQLContent
if req.SQLType == SQLTypeMyBatis {
sql, err = parser.ParseXML(req.SQLContent)
Expand Down Expand Up @@ -128,6 +141,92 @@ func convertTaskResultToAuditResV1(task *model.Task) *AuditResDataV1 {
}
}

type DirectAuditFileReqV1 struct {
InstanceType string `json:"instance_type" form:"instance_type" example:"MySQL" valid:"required"`
// 调用方不应该关心SQL是否被完美的拆分成独立的条目, 拆分SQL由SQLE实现
// 每个数组元素是一个文件内容
FileContents []string `json:"file_contents" form:"file_contents" example:"select * from t1; select * from t2;" valid:"required"`
SQLType string `json:"sql_type" form:"sql_type" example:"sql" enums:"sql,mybatis," valid:"omitempty,oneof=sql mybatis"`
ProjectName string `json:"project_name" form:"project_name" example:"project1" valid:"required"`
InstanceName *string `json:"instance_name" form:"instance_name" example:"instance1"`
SchemaName *string `json:"schema_name" form:"schema_name" example:"schema1"`
}

// @Summary 直接从文件内容提取SQL并审核，SQL文件暂时只支持一次解析一个文件
// @Description Direct audit sql from SQL files and MyBatis files
// @Id directAuditFilesV1
// @Tags sql_audit
// @Security ApiKeyAuth
// @Param req body v1.DirectAuditFileReqV1 true "files that should be audited"
// @Success 200 {object} v1.DirectAuditResV1
// @router /v1/audit_files [post]
func DirectAuditFiles(c echo.Context) error {
req := new(DirectAuditFileReqV1)
err := controller.BindAndValidateReq(c, req)
if err != nil {
return err
}

user := controller.GetUserName(c)
s := model.GetStorage()
if yes, err := s.IsProjectMember(user, req.ProjectName); err != nil {
return controller.JSONBaseErrorReq(c, fmt.Errorf("check privilege failed: %v", err))
} else if !yes {
return controller.JSONBaseErrorReq(c, errors.New(errors.ErrAccessDeniedError, e.New("you are not the project member")))
}

if len(req.FileContents) <= 0 {
return controller.JSONBaseErrorReq(c, e.New("file_contents is required"))
}

sqls := ""
if req.SQLType == SQLTypeMyBatis {
ss, err := parser.ParseXMLs(req.FileContents, false)
if err != nil {
return controller.JSONBaseErrorReq(c, err)
}
sqls = strings.Join(ss, ";")
} else {
// sql文件暂时只支持一次解析一个文件
sqls = req.FileContents[0]
}

l := log.NewEntry().WithField("/v2/audit_files", "direct audit files failed")

var instance *model.Instance
var exist bool
if req.InstanceName != nil {
instance, exist, err = s.GetInstanceByNameAndProjectName(*req.InstanceName, req.ProjectName)
if err != nil {
return controller.JSONBaseErrorReq(c, err)
}
if !exist {
return controller.JSONBaseErrorReq(c, ErrInstanceNotExist)
}
}

var schemaName string
if req.SchemaName != nil {
schemaName = *req.SchemaName
}

var task *model.Task
if instance != nil && schemaName != "" {
task, err = server.DirectAuditByInstance(l, sqls, schemaName, instance)
} else {
task, err = server.AuditSQLByDBType(l, sqls, req.InstanceType, nil, "")
}
if err != nil {
l.Errorf("audit sqls failed: %v", err)
return controller.JSONBaseErrorReq(c, ErrDirectAudit)
}

return c.JSON(http.StatusOK, DirectAuditResV1{
BaseRes: controller.BaseRes{},
Data: convertTaskResultToAuditResV1(task),
})
}

type GetSQLAnalysisReq struct {
ProjectName string `json:"project_name" query:"project_name" example:"default" valid:"required"`
InstanceName string `json:"instance_name" query:"instance_name" example:"MySQL" valid:"required"`
Expand Down
1 change: 1 addition & 0 deletions sqle/api/controller/v2/sql_audit.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ type DirectAuditResV2 struct {
Data *AuditResDataV2 `json:"data"`
}

// @Deprecated
// @Summary 直接审核SQL
// @Description Direct audit sql
// @Id directAuditV2
Expand Down
76 changes: 76 additions & 0 deletions sqle/docs/docs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,40 @@ var doc = `{
"host": "{{.Host}}",
"basePath": "{{.BasePath}}",
"paths": {
"/v1/audit_files": {
"post": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Direct audit sql from SQL files and MyBatis files",
"tags": [
"sql_audit"
],
"summary": "直接从文件内容提取SQL并审核，SQL文件暂时只支持一次解析一个文件",
"operationId": "directAuditFilesV1",
"parameters": [
{
"description": "files that should be audited",
"name": "req",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/v1.DirectAuditFileReqV1"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/v1.DirectAuditResV1"
}
}
}
}
},
"/v1/audit_plan_metas": {
"get": {
"security": [
Expand Down Expand Up @@ -6170,6 +6204,7 @@ var doc = `{
],
"summary": "直接审核SQL",
"operationId": "directAuditV1",
"deprecated": true,
"parameters": [
{
"description": "sqls that should be audited",
Expand Down Expand Up @@ -8823,6 +8858,7 @@ var doc = `{
],
"summary": "直接审核SQL",
"operationId": "directAuditV2",
"deprecated": true,
"parameters": [
{
"description": "sqls that should be audited",
Expand Down Expand Up @@ -10081,6 +10117,46 @@ var doc = `{
}
}
},
"v1.DirectAuditFileReqV1": {
"type": "object",
"properties": {
"file_contents": {
"description": "调用方不应该关心SQL是否被完美的拆分成独立的条目, 拆分SQL由SQLE实现\n每个数组元素是一个文件内容",
"type": "array",
"items": {
"type": "string"
},
"example": [
"select * from t1; select * from t2;"
]
},
"instance_name": {
"type": "string",
"example": "instance1"
},
"instance_type": {
"type": "string",
"example": "MySQL"
},
"project_name": {
"type": "string",
"example": "project1"
},
"schema_name": {
"type": "string",
"example": "schema1"
},
"sql_type": {
"type": "string",
"enum": [
"sql",
"mybatis",
""
],
"example": "sql"
}
}
},
"v1.DirectAuditReqV1": {
"type": "object",
"properties": {
Expand Down
76 changes: 76 additions & 0 deletions sqle/docs/swagger.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,40 @@
},
"basePath": "/",
"paths": {
"/v1/audit_files": {
"post": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Direct audit sql from SQL files and MyBatis files",
"tags": [
"sql_audit"
],
"summary": "直接从文件内容提取SQL并审核，SQL文件暂时只支持一次解析一个文件",
"operationId": "directAuditFilesV1",
"parameters": [
{
"description": "files that should be audited",
"name": "req",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/v1.DirectAuditFileReqV1"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/v1.DirectAuditResV1"
}
}
}
}
},
"/v1/audit_plan_metas": {
"get": {
"security": [
Expand Down Expand Up @@ -6154,6 +6188,7 @@
],
"summary": "直接审核SQL",
"operationId": "directAuditV1",
"deprecated": true,
"parameters": [
{
"description": "sqls that should be audited",
Expand Down Expand Up @@ -8807,6 +8842,7 @@
],
"summary": "直接审核SQL",
"operationId": "directAuditV2",
"deprecated": true,
"parameters": [
{
"description": "sqls that should be audited",
Expand Down Expand Up @@ -10065,6 +10101,46 @@
}
}
},
"v1.DirectAuditFileReqV1": {
"type": "object",
"properties": {
"file_contents": {
"description": "调用方不应该关心SQL是否被完美的拆分成独立的条目, 拆分SQL由SQLE实现\n每个数组元素是一个文件内容",
"type": "array",
"items": {
"type": "string"
},
"example": [
"select * from t1; select * from t2;"
]
},
"instance_name": {
"type": "string",
"example": "instance1"
},
"instance_type": {
"type": "string",
"example": "MySQL"
},
"project_name": {
"type": "string",
"example": "project1"
},
"schema_name": {
"type": "string",
"example": "schema1"
},
"sql_type": {
"type": "string",
"enum": [
"sql",
"mybatis",
""
],
"example": "sql"
}
}
},
"v1.DirectAuditReqV1": {
"type": "object",
"properties": {
Expand Down
Loading
Loading