From 0a2b8ef62c2516fd7f6935e45022de39ef322e63 Mon Sep 17 00:00:00 2001 From: zyguan Date: Thu, 16 Apr 2026 05:18:57 +0000 Subject: [PATCH 1/6] metrics: enhance diagnostic capabilities for gRPC network issues Signed-off-by: zyguan --- DEPS.bzl | 24 ++--- go.mod | 6 +- go.sum | 8 +- pkg/importsdk/BUILD.bazel | 1 + pkg/metrics/BUILD.bazel | 8 +- pkg/metrics/main_test.go | 3 + pkg/metrics/metrics.go | 128 +++++++++++++++++++++++++++ pkg/metrics/metrics_internal_test.go | 92 +++++++++++++++++++ 8 files changed, 250 insertions(+), 20 deletions(-) diff --git a/DEPS.bzl b/DEPS.bzl index 893d5180ed64b..fae55faf80e10 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -6530,13 +6530,13 @@ def go_deps(): name = "com_github_pingcap_kvproto", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/kvproto", - sha256 = "a16a464c372f295061deaf6134a9e26602ebf780beb9d7d3324fb8d1e333eace", - strip_prefix = "github.com/pingcap/kvproto@v0.0.0-20260326084500-678ff92b1edd", + sha256 = "dcd969a369b184787231d8b6600490c23bdd327e3d405da6a6c309bf5ea3b4b1", + strip_prefix = "github.com/pingcap/kvproto@v0.0.0-20260408021215-335c5c64af53", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20260326084500-678ff92b1edd.zip", - "http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20260326084500-678ff92b1edd.zip", - "https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20260326084500-678ff92b1edd.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20260326084500-678ff92b1edd.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20260408021215-335c5c64af53.zip", + "http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20260408021215-335c5c64af53.zip", + "https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20260408021215-335c5c64af53.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20260408021215-335c5c64af53.zip", ], ) go_repository( @@ -7805,13 +7805,13 @@ def go_deps(): build_tags = ["nextgen", "intest"], build_file_proto_mode = "disable_global", importpath = "github.com/tikv/client-go/v2", - sha256 = "46ee8c64e0f95ad1514e824506e49ea1d9f75329ee6419aae4b0817336550557", - strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20260401083018-b7f9a9e9d2ab", + sha256 = "4931d629046c60e96119106a907e473e847fef5e2d15a65253f39e41a95b013a", + strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20260415115642-0eed1ff3c43e", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260401083018-b7f9a9e9d2ab.zip", - "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260401083018-b7f9a9e9d2ab.zip", - "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260401083018-b7f9a9e9d2ab.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260401083018-b7f9a9e9d2ab.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260415115642-0eed1ff3c43e.zip", + "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260415115642-0eed1ff3c43e.zip", + "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260415115642-0eed1ff3c43e.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260415115642-0eed1ff3c43e.zip", ], ) go_repository( diff --git a/go.mod b/go.mod index 62869704e4a28..cab8ee62d6f5f 100644 --- a/go.mod +++ b/go.mod @@ -101,7 +101,7 @@ require ( github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 github.com/pingcap/fn v1.0.0 - github.com/pingcap/kvproto v0.0.0-20260326084500-678ff92b1edd + github.com/pingcap/kvproto v0.0.0-20260408021215-335c5c64af53 github.com/pingcap/log v1.1.1-0.20250917021125-19901e015dc9 github.com/pingcap/metering_sdk v0.0.0-20260324055927-14fead745f1d github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 @@ -124,7 +124,7 @@ require ( github.com/stathat/consistent v1.0.0 github.com/stretchr/testify v1.11.1 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 - github.com/tikv/client-go/v2 v2.0.8-0.20260401083018-b7f9a9e9d2ab + github.com/tikv/client-go/v2 v2.0.8-0.20260415115642-0eed1ff3c43e github.com/tikv/pd/client v0.0.0-20260404141330-8a6813497b52 github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 github.com/twmb/murmur3 v1.1.6 @@ -353,7 +353,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect google.golang.org/protobuf v1.36.10 gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apimachinery v0.29.11 // indirect k8s.io/klog/v2 v2.120.1 // indirect diff --git a/go.sum b/go.sum index 0f7643f533194..ea6ff5380da1c 100644 --- a/go.sum +++ b/go.sum @@ -728,8 +728,8 @@ github.com/pingcap/fn v1.0.0/go.mod h1:u9WZ1ZiOD1RpNhcI42RucFh/lBuzTu6rw88a+oF2Z github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20260326084500-678ff92b1edd h1:FA2DzGly3tuBWFjktkJxmqeOVEqgrsUvKMQXAw9xvWE= -github.com/pingcap/kvproto v0.0.0-20260326084500-678ff92b1edd/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE= +github.com/pingcap/kvproto v0.0.0-20260408021215-335c5c64af53 h1:wjhJRzyeRKpJqMg6XmqQ7cJdhEhE5mSoCh94rWdTVOk= +github.com/pingcap/kvproto v0.0.0-20260408021215-335c5c64af53/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/log v1.1.1-0.20250917021125-19901e015dc9 h1:qG9BSvlWFEE5otQGamuWedx9LRm0nrHvsQRQiW8SxEs= @@ -895,8 +895,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/tikv/client-go/v2 v2.0.8-0.20260401083018-b7f9a9e9d2ab h1:t9Kh7tVsSSUMuSPpHChPF6W3VtN4Kq7Gi8EgWPbYRyY= -github.com/tikv/client-go/v2 v2.0.8-0.20260401083018-b7f9a9e9d2ab/go.mod h1:lfRxHwyBp1rjTmNC04SUZ+dqk7i1R1AeJ2zraMQaNvY= +github.com/tikv/client-go/v2 v2.0.8-0.20260415115642-0eed1ff3c43e h1:YvslQEfuAbak6ube/LDKsDuG3qwYpvUpzYxVZI37GWU= +github.com/tikv/client-go/v2 v2.0.8-0.20260415115642-0eed1ff3c43e/go.mod h1:rg9c3yf9lfQdj9rt5FvwRP9xDubUY6C9hyH4g9zFH1s= github.com/tikv/pd/client v0.0.0-20260404141330-8a6813497b52 h1:fXIMowblD3qdfHXJYGJpe7SbBlTO4S9GPVZZvL3CPG8= github.com/tikv/pd/client v0.0.0-20260404141330-8a6813497b52/go.mod h1:I2yRx/Yf8Y8kgM5f3VNp4a8fWpnjPC4TxWk554AY8bM= github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 h1:9LPGD+jzxMlnk5r6+hJnar67cgpDIz/iyD+rfl5r2Vk= diff --git a/pkg/importsdk/BUILD.bazel b/pkg/importsdk/BUILD.bazel index 3ce7a773851e9..ec05790001cca 100644 --- a/pkg/importsdk/BUILD.bazel +++ b/pkg/importsdk/BUILD.bazel @@ -54,6 +54,7 @@ go_test( "//pkg/lightning/config", "//pkg/lightning/log", "//pkg/lightning/mydump", + "//pkg/parser/ast", "//pkg/parser/mysql", "//pkg/util/table-filter", "@com_github_data_dog_go_sqlmock//:go-sqlmock", diff --git a/pkg/metrics/BUILD.bazel b/pkg/metrics/BUILD.bazel index 139cd300bf0e4..cb2f250aaf932 100644 --- a/pkg/metrics/BUILD.bazel +++ b/pkg/metrics/BUILD.bazel @@ -47,6 +47,12 @@ go_library( "@com_github_prometheus_client_golang//prometheus/collectors", "@com_github_prometheus_client_model//go", "@com_github_tikv_client_go_v2//metrics", + "@com_github_tikv_client_go_v2//util/collectors", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//channelz/grpc_channelz_v1", + "@org_golang_google_grpc//channelz/service", + "@org_golang_google_grpc//credentials/insecure", + "@org_golang_google_grpc//test/bufconn", "@org_uber_go_zap//:zap", ], ) @@ -61,7 +67,7 @@ go_test( ], embed = [":metrics"], flaky = True, - shard_count = 5, + shard_count = 8, deps = [ "//pkg/parser/terror", "//pkg/statistics/handle/cache", diff --git a/pkg/metrics/main_test.go b/pkg/metrics/main_test.go index 8687e2cd2291e..a51e0c3b41462 100644 --- a/pkg/metrics/main_test.go +++ b/pkg/metrics/main_test.go @@ -29,6 +29,9 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.Cleanup(func(int) { + cleanupGrpcChannelzCollectorForTest() + }), } goleak.VerifyTestMain(m, opts...) } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 547d2f5bba0b7..059d50363bff5 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -15,6 +15,8 @@ package metrics import ( + "context" + "net" "sync" "github.com/pingcap/tidb/pkg/dxf/framework/dxfmetric" @@ -25,7 +27,13 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" tikvmetrics "github.com/tikv/client-go/v2/metrics" + tikvcollectors "github.com/tikv/client-go/v2/util/collectors" "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/channelz/grpc_channelz_v1" + "google.golang.org/grpc/channelz/service" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/test/bufconn" ) var ( @@ -394,6 +402,9 @@ func RegisterMetrics() { // StmtSummary prometheus.MustRegister(StmtSummaryWindowRecordCount) prometheus.MustRegister(StmtSummaryWindowEvictedCount) + + // Channelz + setupChannelzCollector() } // Register registers custom collectors. @@ -458,3 +469,120 @@ func ToggleSimplifiedMode(simplified bool) { } } } + +var grpcChannelzCollector struct { + mu sync.Mutex + + listener *bufconn.Listener + server *grpc.Server + conn *grpc.ClientConn + + collector prometheus.Collector + registered bool +} + +func setupChannelzCollector() { + grpcChannelzCollector.mu.Lock() + defer grpcChannelzCollector.mu.Unlock() + + if err := initGrpcChannelzCollectorLocked(); err != nil { + logutil.BgLogger().Error("setup internal channelz collector failed", zap.Error(err)) + return + } + if grpcChannelzCollector.registered { + return + } + prometheus.MustRegister(grpcChannelzCollector.collector) + grpcChannelzCollector.registered = true +} + +// initGrpcChannelzCollectorLocked initializes the singleton channelz collector. +// It must be called with grpcChannelzCollector.mu held. +func initGrpcChannelzCollectorLocked() error { + if grpcChannelzCollector.collector != nil { + return nil + } + + grpcChannelzCollector.listener = bufconn.Listen(1 << 20) + grpcChannelzCollector.server = grpc.NewServer() + service.RegisterChannelzServiceToServer(grpcChannelzCollector.server) + go func(listener *bufconn.Listener, server *grpc.Server) { + if err := server.Serve(listener); err != nil { + logutil.BgLogger().Warn("internal channelz grpc server stopped", zap.Error(err)) + } + }(grpcChannelzCollector.listener, grpcChannelzCollector.server) + + listener := grpcChannelzCollector.listener + conn, err := grpc.NewClient( + "passthrough:///bufnet", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return listener.DialContext(ctx) + }), + ) + if err != nil { + stopGrpcChannelzCollectorLocked() + return err + } + + grpcChannelzCollector.conn = conn + grpcChannelzCollector.collector = tikvcollectors.NewChannelzCollector(conn, channelzCollectorOpts()) + return nil +} + +func channelzCollectorOpts() tikvcollectors.ChannelzCollectorOpts { + return tikvcollectors.ChannelzCollectorOpts{ + Namespace: namespace, + DisableLocalLabel: true, + Filter: func(node any) (collect bool, walkChildren bool) { + // Only collect socket and leaf subchannel info, which are more useful for troubleshooting network issues. + switch n := node.(type) { + case *grpc_channelz_v1.Socket: + return true, false + + case *grpc_channelz_v1.Subchannel: + isLeaf := len(n.GetSocketRef()) > 0 && + len(n.GetChannelRef()) == 0 && + len(n.GetSubchannelRef()) == 0 + + if isLeaf { + return true, true + } + return false, true + + default: + return false, true + } + }, + } +} + +func cleanupGrpcChannelzCollectorForTest() { + grpcChannelzCollector.mu.Lock() + defer grpcChannelzCollector.mu.Unlock() + + stopGrpcChannelzCollectorLocked() +} + +// stopGrpcChannelzCollectorLocked stops and resets the singleton channelz collector. +// It must be called with grpcChannelzCollector.mu held. +func stopGrpcChannelzCollectorLocked() { + if grpcChannelzCollector.registered && grpcChannelzCollector.collector != nil { + prometheus.Unregister(grpcChannelzCollector.collector) + } + if grpcChannelzCollector.conn != nil { + _ = grpcChannelzCollector.conn.Close() + } + if grpcChannelzCollector.server != nil { + grpcChannelzCollector.server.Stop() + } + if grpcChannelzCollector.listener != nil { + _ = grpcChannelzCollector.listener.Close() + } + + grpcChannelzCollector.server = nil + grpcChannelzCollector.listener = nil + grpcChannelzCollector.conn = nil + grpcChannelzCollector.collector = nil + grpcChannelzCollector.registered = false +} diff --git a/pkg/metrics/metrics_internal_test.go b/pkg/metrics/metrics_internal_test.go index 2c5d5a4203d59..da6bc5975e061 100644 --- a/pkg/metrics/metrics_internal_test.go +++ b/pkg/metrics/metrics_internal_test.go @@ -64,3 +64,95 @@ func TestStmtSummaryMetricLabels(t *testing.T) { require.Equal(t, 5.0, readGaugeValue(t, StmtSummaryWindowRecordCount.WithLabelValues(StmtSummaryTypeV2))) require.Equal(t, 2.0, readGaugeValue(t, StmtSummaryWindowEvictedCount.WithLabelValues(StmtSummaryTypeV2))) } + +func TestGrpcChannelzCollectorSingleton(t *testing.T) { + cleanupGrpcChannelzCollectorForTest() + t.Cleanup(cleanupGrpcChannelzCollectorForTest) + + func() { + grpcChannelzCollector.mu.Lock() + defer grpcChannelzCollector.mu.Unlock() + + require.NoError(t, initGrpcChannelzCollectorLocked()) + firstServer := grpcChannelzCollector.server + firstListener := grpcChannelzCollector.listener + firstConn := grpcChannelzCollector.conn + firstCollector := grpcChannelzCollector.collector + + require.NoError(t, initGrpcChannelzCollectorLocked()) + require.Same(t, firstServer, grpcChannelzCollector.server) + require.Same(t, firstListener, grpcChannelzCollector.listener) + require.Same(t, firstConn, grpcChannelzCollector.conn) + require.True(t, firstCollector == grpcChannelzCollector.collector) + }() + + cleanupGrpcChannelzCollectorForTest() + + func() { + grpcChannelzCollector.mu.Lock() + defer grpcChannelzCollector.mu.Unlock() + + require.Nil(t, grpcChannelzCollector.server) + require.Nil(t, grpcChannelzCollector.listener) + require.Nil(t, grpcChannelzCollector.conn) + require.Nil(t, grpcChannelzCollector.collector) + require.False(t, grpcChannelzCollector.registered) + }() +} + +func TestSetupChannelzCollectorRegistersOnce(t *testing.T) { + cleanupGrpcChannelzCollectorForTest() + t.Cleanup(cleanupGrpcChannelzCollectorForTest) + + setupChannelzCollector() + + var firstCollector prometheus.Collector + func() { + grpcChannelzCollector.mu.Lock() + defer grpcChannelzCollector.mu.Unlock() + + firstCollector = grpcChannelzCollector.collector + require.NotNil(t, firstCollector) + require.True(t, grpcChannelzCollector.registered) + }() + + setupChannelzCollector() + + func() { + grpcChannelzCollector.mu.Lock() + defer grpcChannelzCollector.mu.Unlock() + + require.True(t, firstCollector == grpcChannelzCollector.collector) + require.True(t, grpcChannelzCollector.registered) + }() +} + +func TestGrpcChannelzCollectorGather(t *testing.T) { + cleanupGrpcChannelzCollectorForTest() + t.Cleanup(cleanupGrpcChannelzCollectorForTest) + + var collector prometheus.Collector + func() { + grpcChannelzCollector.mu.Lock() + defer grpcChannelzCollector.mu.Unlock() + + require.NoError(t, initGrpcChannelzCollectorLocked()) + collector = grpcChannelzCollector.collector + }() + + registry := prometheus.NewRegistry() + require.NoError(t, registry.Register(collector)) + families, err := registry.Gather() + require.NoError(t, err) + + require.NotNil(t, findMetricFamily(families, "tidb_grpc_channelz_fetch_errors_total")) +} + +func findMetricFamily(families []*dto.MetricFamily, name string) *dto.MetricFamily { + for _, family := range families { + if family.GetName() == name { + return family + } + } + return nil +} From c33b6e113ccff9117819a154fad7589b66982003 Mon Sep 17 00:00:00 2001 From: zyguan Date: Thu, 16 Apr 2026 09:29:32 +0000 Subject: [PATCH 2/6] skip init channelz collector in test Signed-off-by: zyguan --- pkg/metrics/BUILD.bazel | 2 ++ pkg/metrics/metrics.go | 7 ++++++- pkg/metrics/metrics_internal_test.go | 20 +++++--------------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/pkg/metrics/BUILD.bazel b/pkg/metrics/BUILD.bazel index cb2f250aaf932..63f8944438098 100644 --- a/pkg/metrics/BUILD.bazel +++ b/pkg/metrics/BUILD.bazel @@ -40,6 +40,7 @@ go_library( "//pkg/metrics/common", "//pkg/parser/terror", "//pkg/timer/metrics", + "//pkg/util/intest", "//pkg/util/logutil", "//pkg/util/promutil", "@com_github_pingcap_errors//:errors", @@ -72,6 +73,7 @@ go_test( "//pkg/parser/terror", "//pkg/statistics/handle/cache", "//pkg/testkit/testsetup", + "//pkg/util/intest", "@com_github_pingcap_errors//:errors", "@com_github_prometheus_client_golang//prometheus", "@com_github_prometheus_client_model//go", diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 059d50363bff5..ed216ede8d9a5 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/tidb/pkg/ingestor/ingestmetric" metricscommon "github.com/pingcap/tidb/pkg/metrics/common" timermetrics "github.com/pingcap/tidb/pkg/timer/metrics" + "github.com/pingcap/tidb/pkg/util/intest" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" @@ -482,11 +483,15 @@ var grpcChannelzCollector struct { } func setupChannelzCollector() { + if intest.InTest { + return + } + grpcChannelzCollector.mu.Lock() defer grpcChannelzCollector.mu.Unlock() if err := initGrpcChannelzCollectorLocked(); err != nil { - logutil.BgLogger().Error("setup internal channelz collector failed", zap.Error(err)) + logutil.BgLogger().Warn("setup internal channelz collector failed", zap.Error(err)) return } if grpcChannelzCollector.registered { diff --git a/pkg/metrics/metrics_internal_test.go b/pkg/metrics/metrics_internal_test.go index da6bc5975e061..b4d4ba5b22aa2 100644 --- a/pkg/metrics/metrics_internal_test.go +++ b/pkg/metrics/metrics_internal_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/util/intest" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" @@ -100,30 +101,19 @@ func TestGrpcChannelzCollectorSingleton(t *testing.T) { }() } -func TestSetupChannelzCollectorRegistersOnce(t *testing.T) { +func TestSetupChannelzCollectorSkippedInTest(t *testing.T) { cleanupGrpcChannelzCollectorForTest() t.Cleanup(cleanupGrpcChannelzCollectorForTest) + require.True(t, intest.InTest) setupChannelzCollector() - var firstCollector prometheus.Collector func() { grpcChannelzCollector.mu.Lock() defer grpcChannelzCollector.mu.Unlock() - firstCollector = grpcChannelzCollector.collector - require.NotNil(t, firstCollector) - require.True(t, grpcChannelzCollector.registered) - }() - - setupChannelzCollector() - - func() { - grpcChannelzCollector.mu.Lock() - defer grpcChannelzCollector.mu.Unlock() - - require.True(t, firstCollector == grpcChannelzCollector.collector) - require.True(t, grpcChannelzCollector.registered) + require.Nil(t, grpcChannelzCollector.collector) + require.False(t, grpcChannelzCollector.registered) }() } From c5f3b5b7d1a9f2439447eca08e77ddc245e7ed4f Mon Sep 17 00:00:00 2001 From: zyguan Date: Thu, 16 Apr 2026 10:22:01 +0000 Subject: [PATCH 3/6] filter out internal grpc channel Signed-off-by: zyguan --- pkg/metrics/metrics.go | 20 ++++++++++++++++++++ pkg/metrics/metrics_internal_test.go | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index ed216ede8d9a5..80aff2ec1d934 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -542,10 +542,22 @@ func channelzCollectorOpts() tikvcollectors.ChannelzCollectorOpts { Filter: func(node any) (collect bool, walkChildren bool) { // Only collect socket and leaf subchannel info, which are more useful for troubleshooting network issues. switch n := node.(type) { + case *grpc_channelz_v1.Channel: + if isInternalChannelzTarget(n.GetData().GetTarget()) { + return false, false + } + return false, true + case *grpc_channelz_v1.Socket: + if isInternalChannelzSocket(n) { + return false, false + } return true, false case *grpc_channelz_v1.Subchannel: + if isInternalChannelzTarget(n.GetData().GetTarget()) { + return false, false + } isLeaf := len(n.GetSocketRef()) > 0 && len(n.GetChannelRef()) == 0 && len(n.GetSubchannelRef()) == 0 @@ -562,6 +574,14 @@ func channelzCollectorOpts() tikvcollectors.ChannelzCollectorOpts { } } +func isInternalChannelzTarget(target string) bool { + return target == "bufnet" || target == "passthrough:///bufnet" +} + +func isInternalChannelzSocket(socket *grpc_channelz_v1.Socket) bool { + return socket.GetRemote() == nil && socket.GetRemoteName() == "" +} + func cleanupGrpcChannelzCollectorForTest() { grpcChannelzCollector.mu.Lock() defer grpcChannelzCollector.mu.Unlock() diff --git a/pkg/metrics/metrics_internal_test.go b/pkg/metrics/metrics_internal_test.go index b4d4ba5b22aa2..533ab770e3954 100644 --- a/pkg/metrics/metrics_internal_test.go +++ b/pkg/metrics/metrics_internal_test.go @@ -15,6 +15,7 @@ package metrics import ( + "strings" "testing" "github.com/pingcap/errors" @@ -136,6 +137,15 @@ func TestGrpcChannelzCollectorGather(t *testing.T) { require.NoError(t, err) require.NotNil(t, findMetricFamily(families, "tidb_grpc_channelz_fetch_errors_total")) + for _, family := range families { + for _, metric := range family.GetMetric() { + require.False(t, metricHasLabelValue(metric, "target", "bufnet")) + require.False(t, metricHasLabelValue(metric, "target", "passthrough:///bufnet")) + if strings.HasPrefix(family.GetName(), "tidb_grpc_channelz_socket_") { + require.False(t, metricHasLabelValue(metric, "remote", "")) + } + } + } } func findMetricFamily(families []*dto.MetricFamily, name string) *dto.MetricFamily { @@ -146,3 +156,12 @@ func findMetricFamily(families []*dto.MetricFamily, name string) *dto.MetricFami } return nil } + +func metricHasLabelValue(metric *dto.Metric, name string, value string) bool { + for _, label := range metric.GetLabel() { + if label.GetName() == name && label.GetValue() == value { + return true + } + } + return false +} From a6303d88485ed3317d7b9c79ec575951e9ac8d40 Mon Sep 17 00:00:00 2001 From: zyguan Date: Thu, 16 Apr 2026 12:43:01 +0000 Subject: [PATCH 4/6] add internal grpc goroutine to goleak whitelist Signed-off-by: zyguan --- br/cmd/br/main_test.go | 1 + pkg/metrics/main_test.go | 4 +--- pkg/metrics/metrics.go | 12 ++++++------ pkg/server/handler/extractorhandler/main_test.go | 1 + pkg/server/handler/optimizor/main_test.go | 1 + pkg/server/handler/tests/main_test.go | 1 + pkg/server/main_test.go | 1 + pkg/server/tests/commontest/main_test.go | 1 + pkg/server/tests/cursor/main_test.go | 1 + pkg/server/tests/main_test.go | 1 + pkg/server/tests/standby/main_test.go | 1 + pkg/server/tests/tls/main_test.go | 1 + 12 files changed, 17 insertions(+), 9 deletions(-) diff --git a/br/cmd/br/main_test.go b/br/cmd/br/main_test.go index bdd6f7282ee62..c125381526ab6 100644 --- a/br/cmd/br/main_test.go +++ b/br/cmd/br/main_test.go @@ -49,6 +49,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("google.golang.org/grpc.(*ClientConn).WaitForStateChange"), goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/config/retry.(*Config).createBackoffFn.newBackoffFn.func2"), goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer"), + goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), ) } diff --git a/pkg/metrics/main_test.go b/pkg/metrics/main_test.go index a51e0c3b41462..63d0b56c4d99a 100644 --- a/pkg/metrics/main_test.go +++ b/pkg/metrics/main_test.go @@ -29,9 +29,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), - goleak.Cleanup(func(int) { - cleanupGrpcChannelzCollectorForTest() - }), + goleak.Cleanup(func(int) { cleanupGrpcChannelzCollectorForTest() }), } goleak.VerifyTestMain(m, opts...) } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 80aff2ec1d934..76d8c0ef2af9b 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -548,12 +548,6 @@ func channelzCollectorOpts() tikvcollectors.ChannelzCollectorOpts { } return false, true - case *grpc_channelz_v1.Socket: - if isInternalChannelzSocket(n) { - return false, false - } - return true, false - case *grpc_channelz_v1.Subchannel: if isInternalChannelzTarget(n.GetData().GetTarget()) { return false, false @@ -567,6 +561,12 @@ func channelzCollectorOpts() tikvcollectors.ChannelzCollectorOpts { } return false, true + case *grpc_channelz_v1.Socket: + if isInternalChannelzSocket(n) { + return false, false + } + return true, false + default: return false, true } diff --git a/pkg/server/handler/extractorhandler/main_test.go b/pkg/server/handler/extractorhandler/main_test.go index 7a64e8f2fc0ea..f10d5c0931277 100644 --- a/pkg/server/handler/extractorhandler/main_test.go +++ b/pkg/server/handler/extractorhandler/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } goleak.VerifyTestMain(m, opts...) diff --git a/pkg/server/handler/optimizor/main_test.go b/pkg/server/handler/optimizor/main_test.go index b3220444e2d90..004195c872f26 100644 --- a/pkg/server/handler/optimizor/main_test.go +++ b/pkg/server/handler/optimizor/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } goleak.VerifyTestMain(m, opts...) diff --git a/pkg/server/handler/tests/main_test.go b/pkg/server/handler/tests/main_test.go index f35bdc9779f8e..a1e5ff445ad5c 100644 --- a/pkg/server/handler/tests/main_test.go +++ b/pkg/server/handler/tests/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } goleak.VerifyTestMain(m, opts...) diff --git a/pkg/server/main_test.go b/pkg/server/main_test.go index 8f68148f70ed1..732b380b1e582 100644 --- a/pkg/server/main_test.go +++ b/pkg/server/main_test.go @@ -79,6 +79,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("google.golang.org/grpc/internal/transport.(*controlBuffer).get"), goleak.IgnoreTopFunction("google.golang.org/grpc/internal/transport.(*http2Client).keepalive"), goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"), + goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } callback := func(i int) int { diff --git a/pkg/server/tests/commontest/main_test.go b/pkg/server/tests/commontest/main_test.go index 94d058bd8dc86..e20eecf28da82 100644 --- a/pkg/server/tests/commontest/main_test.go +++ b/pkg/server/tests/commontest/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } goleak.VerifyTestMain(m, opts...) diff --git a/pkg/server/tests/cursor/main_test.go b/pkg/server/tests/cursor/main_test.go index 0e6b15755fbd5..87e2ee858a300 100644 --- a/pkg/server/tests/cursor/main_test.go +++ b/pkg/server/tests/cursor/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } goleak.VerifyTestMain(m, opts...) diff --git a/pkg/server/tests/main_test.go b/pkg/server/tests/main_test.go index e4dd64b449d6a..33348b2f68ae1 100644 --- a/pkg/server/tests/main_test.go +++ b/pkg/server/tests/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } goleak.VerifyTestMain(m, opts...) diff --git a/pkg/server/tests/standby/main_test.go b/pkg/server/tests/standby/main_test.go index 83ba9fcc276a6..81518a16bc558 100644 --- a/pkg/server/tests/standby/main_test.go +++ b/pkg/server/tests/standby/main_test.go @@ -69,6 +69,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } goleak.VerifyTestMain(m, opts...) diff --git a/pkg/server/tests/tls/main_test.go b/pkg/server/tests/tls/main_test.go index 7d87e42a738aa..ba50f3689dbfa 100644 --- a/pkg/server/tests/tls/main_test.go +++ b/pkg/server/tests/tls/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } goleak.VerifyTestMain(m, opts...) From 837cd1caa180c4bc2c06b3b09cc45302c70aaaea Mon Sep 17 00:00:00 2001 From: zyguan Date: Thu, 16 Apr 2026 14:38:13 +0000 Subject: [PATCH 5/6] try to fix goleak Signed-off-by: zyguan --- br/cmd/br/main_test.go | 1 + pkg/server/handler/extractorhandler/main_test.go | 1 + pkg/server/handler/optimizor/main_test.go | 1 + pkg/server/handler/tests/main_test.go | 1 + pkg/server/main_test.go | 3 ++- pkg/server/tests/commontest/main_test.go | 1 + pkg/server/tests/cursor/main_test.go | 1 + pkg/server/tests/main_test.go | 1 + pkg/server/tests/standby/main_test.go | 1 + pkg/server/tests/tls/main_test.go | 1 + 10 files changed, 11 insertions(+), 1 deletion(-) diff --git a/br/cmd/br/main_test.go b/br/cmd/br/main_test.go index c125381526ab6..f74d4d3aa1400 100644 --- a/br/cmd/br/main_test.go +++ b/br/cmd/br/main_test.go @@ -49,6 +49,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("google.golang.org/grpc.(*ClientConn).WaitForStateChange"), goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/config/retry.(*Config).createBackoffFn.newBackoffFn.func2"), goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run"), goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), ) } diff --git a/pkg/server/handler/extractorhandler/main_test.go b/pkg/server/handler/extractorhandler/main_test.go index f10d5c0931277..095f9372b2824 100644 --- a/pkg/server/handler/extractorhandler/main_test.go +++ b/pkg/server/handler/extractorhandler/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run"), goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } diff --git a/pkg/server/handler/optimizor/main_test.go b/pkg/server/handler/optimizor/main_test.go index 004195c872f26..a54b1b640aeec 100644 --- a/pkg/server/handler/optimizor/main_test.go +++ b/pkg/server/handler/optimizor/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run"), goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } diff --git a/pkg/server/handler/tests/main_test.go b/pkg/server/handler/tests/main_test.go index a1e5ff445ad5c..e08f866d77685 100644 --- a/pkg/server/handler/tests/main_test.go +++ b/pkg/server/handler/tests/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run"), goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } diff --git a/pkg/server/main_test.go b/pkg/server/main_test.go index 732b380b1e582..ca0daca7c9439 100644 --- a/pkg/server/main_test.go +++ b/pkg/server/main_test.go @@ -78,8 +78,9 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("google.golang.org/grpc.(*ccBalancerWrapper).watcher"), goleak.IgnoreTopFunction("google.golang.org/grpc/internal/transport.(*controlBuffer).get"), goleak.IgnoreTopFunction("google.golang.org/grpc/internal/transport.(*http2Client).keepalive"), - goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run"), goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), + goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"), } callback := func(i int) int { diff --git a/pkg/server/tests/commontest/main_test.go b/pkg/server/tests/commontest/main_test.go index e20eecf28da82..3b13b2d9f7c4e 100644 --- a/pkg/server/tests/commontest/main_test.go +++ b/pkg/server/tests/commontest/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run"), goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } diff --git a/pkg/server/tests/cursor/main_test.go b/pkg/server/tests/cursor/main_test.go index 87e2ee858a300..c34aa9b80e117 100644 --- a/pkg/server/tests/cursor/main_test.go +++ b/pkg/server/tests/cursor/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run"), goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } diff --git a/pkg/server/tests/main_test.go b/pkg/server/tests/main_test.go index 33348b2f68ae1..40ff52fc5e4bd 100644 --- a/pkg/server/tests/main_test.go +++ b/pkg/server/tests/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run"), goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } diff --git a/pkg/server/tests/standby/main_test.go b/pkg/server/tests/standby/main_test.go index 81518a16bc558..6b420333d12b4 100644 --- a/pkg/server/tests/standby/main_test.go +++ b/pkg/server/tests/standby/main_test.go @@ -69,6 +69,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run"), goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } diff --git a/pkg/server/tests/tls/main_test.go b/pkg/server/tests/tls/main_test.go index ba50f3689dbfa..9f445ffb39261 100644 --- a/pkg/server/tests/tls/main_test.go +++ b/pkg/server/tests/tls/main_test.go @@ -64,6 +64,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run"), goleak.IgnoreTopFunction("google.golang.org/grpc/test/bufconn.(*Listener).Accept"), } From 7974954d2efe32ffbfd5fb059d6b35878f94f8ee Mon Sep 17 00:00:00 2001 From: zyguan Date: Fri, 17 Apr 2026 10:16:39 +0000 Subject: [PATCH 6/6] address comment Signed-off-by: zyguan --- pkg/metrics/metrics.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 76d8c0ef2af9b..fccb0677831fc 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -574,10 +574,14 @@ func channelzCollectorOpts() tikvcollectors.ChannelzCollectorOpts { } } +// isInternalChannelzTarget returns true if the target is used for internal channelz collector, which is identified by +// the fact that its target is "bufnet" or "passthrough:///bufnet". func isInternalChannelzTarget(target string) bool { return target == "bufnet" || target == "passthrough:///bufnet" } +// isInternalChannelzSocket returns true if the socket is created by the internal channelz collector for scrapping +// channelz metrics, which is identified by the fact that it has no remote endpoint. func isInternalChannelzSocket(socket *grpc_channelz_v1.Socket) bool { return socket.GetRemote() == nil && socket.GetRemoteName() == "" }