Skip to content

Commit 30f1a6e

Browse files
author
actiontech
committed
Merge branch 'main' of https://github.com/actiontech/sqle into main
2 parents ed09e77 + ca9cddc commit 30f1a6e

File tree

41 files changed

+2620
-3709
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+2620
-3709
lines changed

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.19
55
require (
66
github.com/DATA-DOG/go-sqlmock v1.5.0
77
github.com/Masterminds/semver/v3 v3.1.1
8-
github.com/actiontech/dms v0.0.0-20250731071549-5e4fb4297452
8+
github.com/actiontech/dms v0.0.0-20250821024621-eb0fc701649a
99
github.com/actiontech/java-sql-extractor v0.0.0-20231120024232-2b8c3c0894a1
1010
github.com/actiontech/mybatis-mapper-2-sql v0.5.1-0.20250307064901-aabaceee2249
1111
github.com/agiledragon/gomonkey v2.0.2+incompatible

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc
4646
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
4747
github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
4848
github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
49-
github.com/actiontech/dms v0.0.0-20250731071549-5e4fb4297452 h1:jNW3+Dg2SjM/F/ticZ3+7LSOTVgo7jF7JhsQ/5j4oVw=
50-
github.com/actiontech/dms v0.0.0-20250731071549-5e4fb4297452/go.mod h1:Jm+tDoBYmX8R8zqgPJRq1NwZXdxO/F+HxVRJVF65WXA=
49+
github.com/actiontech/dms v0.0.0-20250821024621-eb0fc701649a h1:A1VYR3c16h9a3iDz2EEfRROc561Zspm2OYrxFZTY3cg=
50+
github.com/actiontech/dms v0.0.0-20250821024621-eb0fc701649a/go.mod h1:Jm+tDoBYmX8R8zqgPJRq1NwZXdxO/F+HxVRJVF65WXA=
5151
github.com/actiontech/java-sql-extractor v0.0.0-20231120024232-2b8c3c0894a1 h1:/UHUgr+645xeXCqXzvOc3IqwOK5gLM+bU/C+s2Xe/Kk=
5252
github.com/actiontech/java-sql-extractor v0.0.0-20231120024232-2b8c3c0894a1/go.mod h1:adDZHhAf2LRMx2h0JzofPXn12x2XlyQjVE116KXquwo=
5353
github.com/actiontech/mybatis-mapper-2-sql v0.5.1-0.20250307064901-aabaceee2249 h1:BjbL364Bl45WMARnEqZrT7Pd0rMhd4vvnQbMvxX3iQw=

sqle/api/app.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,8 +134,6 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config *config.SqleOpti
134134
v1Router.GET("/configurations/ding_talk", v1.GetDingTalkConfigurationV1, sqleMiddleware.ViewGlobalAllowed())
135135
v1Router.PATCH("/configurations/ding_talk", v1.UpdateDingTalkConfigurationV1, sqleMiddleware.OpGlobalAllowed())
136136
v1Router.POST("/configurations/ding_talk/test", v1.TestDingTalkConfigV1, sqleMiddleware.OpGlobalAllowed())
137-
v1Router.GET("/configurations/system_variables", v1.GetSystemVariables, sqleMiddleware.ViewGlobalAllowed())
138-
v1Router.PATCH("/configurations/system_variables", v1.UpdateSystemVariables, sqleMiddleware.OpGlobalAllowed())
139137
v1Router.GET("/configurations/license", v1.GetLicense, sqleMiddleware.ViewGlobalAllowed())
140138
v1Router.POST("/configurations/license", v1.SetLicense, sqleMiddleware.OpGlobalAllowed())
141139
v1Router.GET("/configurations/license/info", v1.GetSQLELicenseInfo, sqleMiddleware.ViewGlobalAllowed())
@@ -414,6 +412,9 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config *config.SqleOpti
414412

415413
v2ProjectOpRouter.POST("/:project_name/audit_plans/:audit_plan_name/full", DeprecatedBy(apiV2))
416414
v2ProjectOpRouter.POST("/:project_name/audit_plans/:audit_plan_name/sqls/partial", DeprecatedBy(apiV2))
415+
416+
// sql optimization
417+
v2ProjectOpRouter.POST("/:project_name/sql_optimization_records", v2.SQLOptimize)
417418
}
418419

419420
v3ProjectOpRouter := v3Router.Group("/projects", sqleMiddleware.ProjectMemberOpAllowed())
@@ -443,6 +444,10 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config *config.SqleOpti
443444
v2ProjectViewRouter.GET("/:project_name/sql_manages/exports", v2.ExportSqlManagesV2)
444445
v2ProjectViewRouter.GET("/:project_name/instance_audit_plans", v2.GetInstanceAuditPlans)
445446
v2ProjectViewRouter.GET("/:project_name/instance_audit_plans/:instance_audit_plan_id", v2.GetInstanceAuditPlanDetail)
447+
448+
// sql optimization
449+
v2ProjectViewRouter.GET("/:project_name/sql_optimization_records", v2.GetOptimizationRecords)
450+
v2ProjectViewRouter.GET("/:project_name/sql_optimization_records/:optimization_record_id/detail", v2.GetOptimizationSQLDetail)
446451
}
447452

448453
{

sqle/api/controller/v1/analysis.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ type AnalysisResult struct {
2222
AffectRowsResultErr error
2323
}
2424

25-
func GetSQLAnalysisResult(l *logrus.Entry, instance *model.Instance, schema, sql string) (res *AnalysisResult, err error) {
25+
func GetSQLAnalysisResult(l *logrus.Entry, instance *model.Instance, schema, sql string, affectRowsEnabled bool) (res *AnalysisResult, err error) {
2626
dsn, err := common.NewDSN(instance, schema)
2727
if err != nil {
2828
return nil, err
@@ -41,7 +41,11 @@ func GetSQLAnalysisResult(l *logrus.Entry, instance *model.Instance, schema, sql
4141
res.Cost = &cost
4242
res.ExplainResult, res.ExplainResultErr = Explain(instance.DbType, plugin, sql)
4343
res.TableMetaResult, res.TableMetaResultErr = GetTableMetas(instance.DbType, plugin, sql)
44-
res.AffectRowsResult, res.AffectRowsResultErr = GetRowsAffected(instance.DbType, plugin, sql)
44+
45+
// 只有当AffectRowsEnabled为true时才执行影响行数计算
46+
if affectRowsEnabled {
47+
res.AffectRowsResult, res.AffectRowsResultErr = GetRowsAffected(instance.DbType, plugin, sql)
48+
}
4549

4650
return res, nil
4751
}

sqle/api/controller/v1/configuration.go

Lines changed: 13 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,21 @@ import (
44
"context"
55
"crypto/rsa"
66
"fmt"
7-
"net/http"
8-
"os"
9-
"strconv"
10-
7+
"github.com/actiontech/dms/pkg/dms-common/dmsobject"
118
"github.com/actiontech/sqle/sqle/errors"
129
"github.com/actiontech/sqle/sqle/utils"
1310
"github.com/go-git/go-git/v5/config"
1411
"github.com/go-git/go-git/v5/plumbing"
12+
"net/http"
13+
"os"
1514

1615
"github.com/actiontech/sqle/sqle/api/controller"
16+
"github.com/actiontech/sqle/sqle/dms"
1717
"github.com/actiontech/sqle/sqle/driver"
1818

19-
"github.com/actiontech/sqle/sqle/model"
20-
2119
"github.com/go-git/go-git/v5"
2220
"github.com/go-git/go-git/v5/plumbing/transport"
2321
goGitTransport "github.com/go-git/go-git/v5/plumbing/transport/http"
24-
2522
sshTransport "github.com/go-git/go-git/v5/plumbing/transport/ssh"
2623
"github.com/labstack/echo/v4"
2724
"golang.org/x/crypto/ssh"
@@ -98,53 +95,6 @@ type UpdateSystemVariablesReqV1 struct {
9895
CbOperationLogsExpiredHours *int `json:"cb_operation_logs_expired_hours" form:"cb_operation_logs_expired_hours" example:"2160"`
9996
}
10097

101-
// @Summary 修改系统变量
102-
// @Description update system variables
103-
// @Accept json
104-
// @Id updateSystemVariablesV1
105-
// @Tags configuration
106-
// @Security ApiKeyAuth
107-
// @Param instance body v1.UpdateSystemVariablesReqV1 true "update system variables request"
108-
// @Success 200 {object} controller.BaseRes
109-
// @router /v1/configurations/system_variables [patch]
110-
func UpdateSystemVariables(c echo.Context) error {
111-
req := new(UpdateSystemVariablesReqV1)
112-
if err := controller.BindAndValidateReq(c, req); err != nil {
113-
return err
114-
}
115-
116-
s := model.GetStorage()
117-
118-
var systemVariables []model.SystemVariable
119-
120-
if req.OperationRecordExpiredHours != nil {
121-
systemVariables = append(systemVariables, model.SystemVariable{
122-
Key: model.SystemVariableOperationRecordExpiredHours,
123-
Value: strconv.Itoa(*req.OperationRecordExpiredHours),
124-
})
125-
}
126-
127-
if req.CbOperationLogsExpiredHours != nil {
128-
systemVariables = append(systemVariables, model.SystemVariable{
129-
Key: model.SystemVariableCbOperationLogsExpiredHours,
130-
Value: strconv.Itoa(*req.CbOperationLogsExpiredHours),
131-
})
132-
}
133-
134-
if req.Url != nil {
135-
systemVariables = append(systemVariables, model.SystemVariable{
136-
Key: model.SystemVariableSqleUrl,
137-
Value: *req.Url,
138-
})
139-
}
140-
141-
if err := s.PathSaveSystemVariables(systemVariables); err != nil {
142-
return controller.JSONBaseErrorReq(c, err)
143-
}
144-
145-
return controller.JSONBaseErrorReq(c, nil)
146-
}
147-
14898
type GetSystemVariablesResV1 struct {
14999
controller.BaseRes
150100
Data SystemVariablesResV1 `json:"data"`
@@ -156,39 +106,6 @@ type SystemVariablesResV1 struct {
156106
CbOperationLogsExpiredHours int `json:"cb_operation_logs_expired_hours"`
157107
}
158108

159-
// @Summary 获取系统变量
160-
// @Description get system variables
161-
// @Id getSystemVariablesV1
162-
// @Tags configuration
163-
// @Security ApiKeyAuth
164-
// @Success 200 {object} v1.GetSystemVariablesResV1
165-
// @router /v1/configurations/system_variables [get]
166-
func GetSystemVariables(c echo.Context) error {
167-
s := model.GetStorage()
168-
systemVariables, err := s.GetAllSystemVariables()
169-
if err != nil {
170-
return controller.JSONBaseErrorReq(c, err)
171-
}
172-
operationRecordExpiredHours, err := strconv.Atoi(systemVariables[model.SystemVariableOperationRecordExpiredHours].Value)
173-
if err != nil {
174-
return controller.JSONBaseErrorReq(c, err)
175-
}
176-
177-
cbOperationLogsExpiredHours, err := strconv.Atoi(systemVariables[model.SystemVariableCbOperationLogsExpiredHours].Value)
178-
if err != nil {
179-
return controller.JSONBaseErrorReq(c, err)
180-
}
181-
182-
return c.JSON(http.StatusOK, &GetSystemVariablesResV1{
183-
BaseRes: controller.NewBaseReq(nil),
184-
Data: SystemVariablesResV1{
185-
Url: systemVariables[model.SystemVariableSqleUrl].Value,
186-
OperationRecordExpiredHours: operationRecordExpiredHours,
187-
CbOperationLogsExpiredHours: cbOperationLogsExpiredHours,
188-
},
189-
})
190-
}
191-
192109
type GetDriversResV1 struct {
193110
controller.BaseRes
194111
Data DriversResV1 `json:"data"`
@@ -498,15 +415,14 @@ func getGitAuthMethod(url, username, password string) (transport.AuthMethod, err
498415
// 2. 查看公钥
499416
// 3. 仓库配置密钥
500417
// 不支持该步骤,用户手动执行
501-
storage := model.GetStorage()
502-
privateKey, exists, err := storage.GetSystemVariableByKey(model.SystemVariableSSHPrimaryKey)
418+
systemVariable, err := dmsobject.GetSystemVariables(context.TODO(), dms.GetDMSServerAddress())
503419
if err != nil {
504420
return nil, err
505421
}
506-
if !exists {
422+
if systemVariable.Code != 0 {
507423
return nil, errors.New(errors.DataNotExist, fmt.Errorf("git ssh private key not found"))
508424
}
509-
publicKeys, err := sshTransport.NewPublicKeys("git", []byte(privateKey.Value), "")
425+
publicKeys, err := sshTransport.NewPublicKeys("git", []byte(systemVariable.Data.SystemVariableSSHPrimaryKey), "")
510426
if err != nil {
511427
return nil, fmt.Errorf("failed to load SSH key: %w", err)
512428
}
@@ -630,20 +546,19 @@ type SSHPublicKeyInfo struct {
630546
// @Success 200 {object} v1.SSHPublicKeyInfoV1Rsp
631547
// @Router /v1/configurations/ssh_key [get]
632548
func GetSSHPublicKey(c echo.Context) error {
633-
storage := model.GetStorage()
634-
systemVariables, exists, err := storage.GetSystemVariableByKey(model.SystemVariableSSHPrimaryKey)
549+
systemVariable, err := dmsobject.GetSystemVariables(context.TODO(), dms.GetDMSServerAddress())
635550
if err != nil {
636551
return controller.JSONBaseErrorReq(c, err)
637552
}
638-
if !exists {
553+
if systemVariable.Code != 0 {
639554
return c.JSON(http.StatusOK, SSHPublicKeyInfoV1Rsp{
640555
BaseRes: controller.NewBaseReq(nil),
641556
Data: SSHPublicKeyInfo{
642557
PublicKey: "",
643558
},
644559
})
645560
}
646-
privateKey, err := ssh.ParseRawPrivateKey([]byte(systemVariables.Value))
561+
privateKey, err := ssh.ParseRawPrivateKey([]byte(systemVariable.Data.SystemVariableSSHPrimaryKey))
647562
if err != nil {
648563
return controller.JSONBaseErrorReq(c, err)
649564
}
@@ -673,14 +588,11 @@ func GetSSHPublicKey(c echo.Context) error {
673588
// @Success 200 {object} controller.BaseRes
674589
// @Router /v1/configurations/ssh_key [post]
675590
func GenSSHKey(c echo.Context) error {
676-
storage := model.GetStorage()
677-
_, exists, err := storage.GetSystemVariableByKey(model.SystemVariableSSHPrimaryKey)
591+
systemVariable, err := dmsobject.GetSystemVariables(context.TODO(), dms.GetDMSServerAddress())
678592
if err != nil {
679593
return controller.JSONBaseErrorReq(c, err)
680594
}
681-
682-
// 如果已存在密钥,则不重新生成
683-
if exists {
595+
if systemVariable.Data.SystemVariableSSHPrimaryKey != "" {
684596
return controller.JSONBaseErrorReq(c, nil)
685597
}
686598

@@ -691,12 +603,7 @@ func GenSSHKey(c echo.Context) error {
691603
}
692604

693605
// 保存密钥到系统变量
694-
err = storage.PathSaveSystemVariables([]model.SystemVariable{
695-
{
696-
Key: model.SystemVariableSSHPrimaryKey,
697-
Value: primaryKey,
698-
},
699-
})
606+
err = dms.UpdateSystemVariables(c.Request().Context(), dms.SystemVariableSSHPrimaryKey, primaryKey)
700607
if err != nil {
701608
return controller.JSONBaseErrorReq(c, err)
702609
}

sqle/api/controller/v1/instance_audit_plan.go

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"net/http"
1111
"net/url"
1212
"strconv"
13+
"strings"
1314
"time"
1415

1516
v1 "github.com/actiontech/dms/pkg/dms-common/api/dms/v1"
@@ -1252,12 +1253,19 @@ func ConvertReqToAuditPlanFilter(fs []Filter) []auditplan.Filter {
12521253
// @Param project_name path string true "project name"
12531254
// @Param instance_audit_plan_id path string true "instance audit plan id"
12541255
// @Param id path string true "audit plan sql id"
1256+
// @Param affectRowsEnabled query bool false "whether to calculate and return affected rows, default is true"
12551257
// @Security ApiKeyAuth
12561258
// @Success 200 {object} v1.GetSqlManageSqlAnalysisResp
12571259
// @router /v1/projects/{project_name}/instance_audit_plans/{instance_audit_plan_id}/sqls/{id}/analysis [get]
12581260
func GetAuditPlanSqlAnalysisData(c echo.Context) error {
12591261
insAuditPlanID := c.Param("instance_audit_plan_id")
12601262
sqlManageRecordId := c.Param("id")
1263+
affectRowsEnabled := true
1264+
if affectRowsEnabledStr := c.QueryParam("affectRowsEnabled"); affectRowsEnabledStr != "" {
1265+
if strings.EqualFold(affectRowsEnabledStr, "false") {
1266+
affectRowsEnabled = false
1267+
}
1268+
}
12611269
projectUID, err := dms.GetProjectUIDByName(c.Request().Context(), c.Param("project_name"))
12621270
if err != nil {
12631271
return controller.JSONBaseErrorReq(c, err)
@@ -1285,14 +1293,14 @@ func GetAuditPlanSqlAnalysisData(c echo.Context) error {
12851293
return controller.JSONBaseErrorReq(c, err)
12861294
}
12871295

1288-
res, err := GetSQLAnalysisResult(log.NewEntry(), instance, originSQL.SchemaName, originSQL.SqlText)
1296+
res, err := GetSQLAnalysisResult(log.NewEntry(), instance, originSQL.SchemaName, originSQL.SqlText, affectRowsEnabled)
12891297
if err != nil {
12901298
return controller.JSONBaseErrorReq(c, err)
12911299
}
12921300

12931301
return c.JSON(http.StatusOK, &GetSqlManageSqlAnalysisResp{
12941302
BaseRes: controller.NewBaseReq(nil),
1295-
Data: convertSQLAnalysisResultToRes(c.Request().Context(), res, originSQL.SqlText),
1303+
Data: convertSQLAnalysisResultToRes(c.Request().Context(), res, originSQL.SqlText, affectRowsEnabled),
12961304
})
12971305
}
12981306

sqle/api/controller/v1/sql_audit_record.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -110,18 +110,18 @@ func CreateSQLAuditRecord(c echo.Context) error {
110110
ruleTemplateID = ruleTemplate.ID
111111
}
112112

113-
sqls := getSQLFromFileResp{}
113+
sqls := GetSQLFromFileResp{}
114114
user, err := controller.GetCurrentUser(c, dms.GetUser)
115115
if err != nil {
116116
return controller.JSONBaseErrorReq(c, err)
117117
}
118118
if req.Sqls != "" {
119-
sqls = getSQLFromFileResp{
119+
sqls = GetSQLFromFileResp{
120120
SourceType: model.TaskSQLSourceFromFormData,
121121
SQLsFromFormData: req.Sqls,
122122
}
123123
} else {
124-
sqls, err = getSQLFromFile(c)
124+
sqls, err = GetSQLFromFile(c)
125125
if err != nil {
126126
return controller.JSONBaseErrorReq(c, err)
127127
}
@@ -195,7 +195,7 @@ type TestGitConnectionResDataV1 struct {
195195
ErrorMessage string `json:"error_message,omitempty"`
196196
}
197197

198-
type getSQLFromFileResp struct {
198+
type GetSQLFromFileResp struct {
199199
SourceType string
200200
SQLsFromFormData string
201201
SQLsFromSQLFiles []SQLsFromSQLFile
@@ -213,7 +213,7 @@ type SQLFromXML struct {
213213
SQL string
214214
}
215215

216-
func (s getSQLFromFileResp) MergeSQLs() (sqls string) {
216+
func (s GetSQLFromFileResp) MergeSQLs() (sqls string) {
217217
for _, v := range s.SQLsFromSQLFiles {
218218
sqls += v.SQLs
219219
}
@@ -223,7 +223,7 @@ func (s getSQLFromFileResp) MergeSQLs() (sqls string) {
223223
return sqls
224224
}
225225

226-
func addSQLsFromFileToTasks(sqls getSQLFromFileResp, task *model.Task, plugin driver.Plugin) error {
226+
func addSQLsFromFileToTasks(sqls GetSQLFromFileResp, task *model.Task, plugin driver.Plugin) error {
227227
var num uint = 1
228228

229229
fileTask := func(sqlsText, filePath string, defaultStartLine uint64) error {
@@ -275,7 +275,7 @@ func addSQLsFromFileToTasks(sqls getSQLFromFileResp, task *model.Task, plugin dr
275275
return nil
276276
}
277277

278-
func buildOnlineTaskForAudit(c echo.Context, s *model.Storage, userId uint64, instanceName, instanceSchema, projectUid string, sqls getSQLFromFileResp) (*model.Task, error) {
278+
func buildOnlineTaskForAudit(c echo.Context, s *model.Storage, userId uint64, instanceName, instanceSchema, projectUid string, sqls GetSQLFromFileResp) (*model.Task, error) {
279279
instance, exist, err := dms.GetInstanceInProjectByName(c.Request().Context(), projectUid, instanceName)
280280
if err != nil {
281281
return nil, err
@@ -320,7 +320,7 @@ func buildOnlineTaskForAudit(c echo.Context, s *model.Storage, userId uint64, in
320320
return task, nil
321321
}
322322

323-
func buildOfflineTaskForAudit(userId uint64, dbType string, sqls getSQLFromFileResp) (*model.Task, error) {
323+
func buildOfflineTaskForAudit(userId uint64, dbType string, sqls GetSQLFromFileResp) (*model.Task, error) {
324324
task := &model.Task{
325325
CreateUserId: userId,
326326
ExecuteSQLs: []*model.ExecuteSQL{},

0 commit comments

Comments
 (0)