Skip to content

feat(auditplan): add CatalogName field to schema_meta for PG-family databases #3315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
actiontech-bot wants to merge 1 commit into from
Closed

feat(auditplan): add CatalogName field to schema_meta for PG-family databases #3315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions sqle/locale/active.en.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ ApMetricNameLogicReadPageTotal = "Total logical read pages"
ApMetricNameMaxQueryTime = "Max execution time"
ApMetricNameMemMax = "Maximum memory used"
ApMetricNameMemorySizeTotal = "Total memory consumption(MB)"
ApMetricNameMetaCatalog = "Catalog"
ApMetricNameMetaName = "Object name"
ApMetricNameMetaType = "Object type"
ApMetricNameNoIndexUsedTotal = "Unused-index count"
Expand Down
1 change: 1 addition & 0 deletions sqle/locale/active.zh.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ ApMetricNameLogicReadPageTotal = "逻辑读页数"
ApMetricNameMaxQueryTime = "最长执行时间(s)"
ApMetricNameMemMax = "使用的最大内存空间"
ApMetricNameMemorySizeTotal = "总内存消耗(MB)"
ApMetricNameMetaCatalog = "catalog"
ApMetricNameMetaName = "对象名称"
ApMetricNameMetaType = "对象类型"
ApMetricNameNoIndexUsedTotal = "累计未使用索引次数"
Expand Down
1 change: 1 addition & 0 deletions sqle/locale/message_zh.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,7 @@ var (
ApMetricNameHost = &i18n.Message{ID: "ApMetricNameHost", Other: "主机"}
ApMetricNameMetaName = &i18n.Message{ID: "ApMetricNameMetaName", Other: "对象名称"}
ApMetricNameMetaType = &i18n.Message{ID: "ApMetricNameMetaType", Other: "对象类型"}
ApMetricNameMetaCatalog = &i18n.Message{ID: "ApMetricNameMetaCatalog", Other: "catalog"}
ApMetricNameQueryTimeTotal = &i18n.Message{ID: "ApMetricNameQueryTimeTotal", Other: "总执行时间(s)"}
ApMetricNameQueryTimeAvg = &i18n.Message{ID: "ApMetricNameQueryTimeAvg", Other: "平均执行时间(s)"}
ApMetricNameQueryTimeTotalMS = &i18n.Message{ID: "ApMetricNameQueryTimeTotalMS", Other: "总执行时间(ms)"}
Expand Down
8 changes: 5 additions & 3 deletions sqle/server/auditplan/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@ const MetricNameHost string = "host"
const MetricNameEndpoints string = "endpoints"
const MetricNameStartTimeOfLastScrapedSQL string = "start_time_of_last_scraped_sql" // 抓取sql的开始时间

const MetricNameMetaName string = "schema_meta_name" // 表或者视图的名字
const MetricNameMetaType string = "schema_meta_type" // 表或者视图等等
const MetricNameRecordDeleted string = "record_deleted" // 标记记录是否被删除掉
const MetricNameMetaName string = "schema_meta_name" // 表或者视图的名字
const MetricNameMetaType string = "schema_meta_type" // 表或者视图等等
const MetricNameMetaCatalog string = "schema_meta_catalog" // 对象所在的 catalog（PG 系语义下为 database 名；无 catalog 概念的数据库为空）
const MetricNameRecordDeleted string = "record_deleted" // 标记记录是否被删除掉

const MetricNameQueryTimeTotal string = "query_time_total" // 总执行时间
const MetricNameCPUTimeAvg string = "cpu_time_avg"
Expand Down Expand Up @@ -96,6 +97,7 @@ var ALLMetric = map[string]MetricType{
MetricNameStartTimeOfLastScrapedSQL: MetricTypeString, // MySQL slow log
MetricNameMetaName: MetricTypeString, // MySQL schema meta
MetricNameMetaType: MetricTypeString, // MySQL schema meta
MetricNameMetaCatalog: MetricTypeString, // PG-family schema meta (catalog/database name)
MetricNameRecordDeleted: MetricTypeBool, // MySQL schema meta
MetricNameHost: MetricTypeString, // OB MySQL Full Collect

Expand Down
26 changes: 22 additions & 4 deletions sqle/server/auditplan/task_type_mysql_schema_meta.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,10 +122,16 @@ type BaseSchemaMetaTaskV2 struct {
}

type SchemaMetaSQL struct {
// SchemaName 表示对象所在的逻辑 schema/owner（PostgreSQL 系语义下指 namespace 名，例如 public、app_a）。
// 对于无 catalog/schema 区分的数据库（MySQL、TiDB 等），SchemaName 保留为采集到的库名即可。
SchemaName string
MetaName string
MetaType string
SQLContent string
// CatalogName 表示对象所在的 catalog（PostgreSQL 系语义下指 database 名，例如 db_a）。
// 仅当目标数据库存在 catalog 与 schema 双层命名空间时（PG / TBase / TDSQL-PG / openGauss / GaussDB 等）填充；
// 其它数据库可留空字符串。
CatalogName string
MetaName string
MetaType string
SQLContent string
}

func (at *BaseSchemaMetaTaskV2) Params(instanceId ...string) params.Params {
Expand Down Expand Up @@ -155,6 +161,7 @@ func (at *BaseSchemaMetaTaskV2) Metrics() []string {
return []string{
MetricNameMetaType,
MetricNameMetaName,
MetricNameMetaCatalog,
MetricNameRecordDeleted,
}
}
Expand All @@ -163,6 +170,7 @@ func (at *BaseSchemaMetaTaskV2) genSQLId(sql *SQLV2) string {
md5Json, err := json.Marshal(
struct {
ProjectId string
Catalog string
Schema string
InstName string
Source string
Expand All @@ -171,6 +179,7 @@ func (at *BaseSchemaMetaTaskV2) genSQLId(sql *SQLV2) string {
MetaType string
}{
ProjectId: sql.ProjectId,
Catalog: sql.Info.Get(MetricNameMetaCatalog).String(),
Schema: sql.SchemaName,
InstName: sql.InstanceID,
Source: sql.Source,
Expand All @@ -180,7 +189,7 @@ func (at *BaseSchemaMetaTaskV2) genSQLId(sql *SQLV2) string {
},
)
if err != nil { // todo: 处理错误
return utils.Md5String(fmt.Sprintf("%s:%s:%s:%s", sql.AuditPlanId, sql.SchemaName, sql.Info.Get(MetricNameMetaName).String(), sql.Info.Get(MetricNameMetaType).String()))
return utils.Md5String(fmt.Sprintf("%s:%s:%s:%s:%s", sql.AuditPlanId, sql.Info.Get(MetricNameMetaCatalog).String(), sql.SchemaName, sql.Info.Get(MetricNameMetaName).String(), sql.Info.Get(MetricNameMetaType).String()))
} else {
return utils.Md5String(string(md5Json))
}
Expand All @@ -200,6 +209,9 @@ func (at *BaseSchemaMetaTaskV2) mergeSQL(originSQL, mergedSQL *SQLV2) {
// meta name
originSQL.Info.SetString(MetricNameMetaName, mergedSQL.Info.Get(MetricNameMetaName).String())

// meta catalog（PG 系 catalog/database 名，无 catalog 概念的数据库为空字符串）
originSQL.Info.SetString(MetricNameMetaCatalog, mergedSQL.Info.Get(MetricNameMetaCatalog).String())

// record deleted
originSQL.Info.SetBool(MetricNameRecordDeleted, mergedSQL.Info.Get(MetricNameRecordDeleted).Bool())

Expand Down Expand Up @@ -251,6 +263,7 @@ func (at *BaseSchemaMetaTaskV2) ExtractSQL(logger *logrus.Entry, ap *AuditPlan,
sqlV2.Info.SetBool(MetricNameRecordDeleted, false)
sqlV2.Info.SetString(MetricNameMetaName, sql.MetaName)
sqlV2.Info.SetString(MetricNameMetaType, sql.MetaType)
sqlV2.Info.SetString(MetricNameMetaCatalog, sql.CatalogName)
sqlV2.SQLId = at.genSQLId(sqlV2)
if err := at.AggregateSQL(cache, sqlV2); err != nil {
logger.Errorf("aggregate sql failed, error: %v", err)
Expand Down Expand Up @@ -292,6 +305,10 @@ func (at *BaseSchemaMetaTaskV2) Head(ap *AuditPlan) []Head {
Name: model.AuditResultName,
Desc: model.AuditResultDesc,
},
{
Name: MetricNameMetaCatalog,
Desc: locale.ApMetricNameMetaCatalog,
},
{
Name: "schema_name",
Desc: locale.ApSchema,
Expand Down Expand Up @@ -360,6 +377,7 @@ func (at *BaseSchemaMetaTaskV2) GetSQLData(ctx context.Context, ap *AuditPlan, p
"id": sql.AuditPlanSqlId,
MetricNameMetaName: info.Get(MetricNameMetaName).String(),
MetricNameMetaType: info.Get(MetricNameMetaType).String(),
MetricNameMetaCatalog: info.Get(MetricNameMetaCatalog).String(),
model.AuditResultName: sql.AuditResult.GetAuditJsonStrByLangTag(locale.Bundle.GetLangTagFromCtx(ctx)),
model.AuditStatus: sql.AuditStatus,
})
Expand Down
Loading