diff --git a/BUILD.bazel b/BUILD.bazel index 16fdf970b..4c825a3c5 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "//cache/disk:go_default_library", "//config:go_default_library", "//ldap:go_default_library", + "//otel:go_default_library", "//server:go_default_library", "//utils/flags:go_default_library", "//utils/idle:go_default_library", @@ -28,6 +29,8 @@ go_library( "@com_github_slok_go_http_metrics//middleware:go_default_library", "@com_github_slok_go_http_metrics//middleware/std:go_default_library", "@com_github_urfave_cli_v2//:go_default_library", + "@io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc//:go_default_library", + "@io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp//:go_default_library", "@org_golang_google_grpc//:go_default_library", "@org_golang_google_grpc//credentials:go_default_library", "@org_golang_x_sync//errgroup:go_default_library", diff --git a/MODULE.bazel b/MODULE.bazel index c190b214b..e8e1953a7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -128,6 +128,11 @@ use_repo( "in_gopkg_mgo_v2", "in_gopkg_yaml_v3", "io_etcd_go_bbolt", + "io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc", + "io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp", + "io_opentelemetry_go_otel", + "io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracegrpc", + "io_opentelemetry_go_otel_sdk", "org_golang_google_genproto_googleapis_api", "org_golang_google_genproto_googleapis_bytestream", "org_golang_google_genproto_googleapis_rpc", diff --git a/cache/disk/disk_test.go b/cache/disk/disk_test.go index d3eebb90c..d0b88a46a 100644 --- a/cache/disk/disk_test.go +++ b/cache/disk/disk_test.go @@ -890,7 +890,7 @@ func TestHttpProxyBackend(t *testing.T) { accessLogger := testutils.NewSilentLogger() errorLogger := testutils.NewSilentLogger() - proxy, err := httpproxy.New(url, "zstd", &http.Client{}, accessLogger, errorLogger, 100, 1000000) + proxy, err := httpproxy.New(url, "zstd", &http.Client{}, accessLogger, errorLogger, 100, 1000000, false) if err != nil { t.Fatal(err) } diff --git a/cache/gcsproxy/BUILD.bazel b/cache/gcsproxy/BUILD.bazel index caf019164..eff609161 100644 --- a/cache/gcsproxy/BUILD.bazel +++ b/cache/gcsproxy/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//cache:go_default_library", "//cache/httpproxy:go_default_library", + "@io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp//:go_default_library", "@org_golang_x_oauth2//:go_default_library", "@org_golang_x_oauth2//google:go_default_library", ], diff --git a/cache/gcsproxy/gcsproxy.go b/cache/gcsproxy/gcsproxy.go index 1c86b73b1..0543b6260 100644 --- a/cache/gcsproxy/gcsproxy.go +++ b/cache/gcsproxy/gcsproxy.go @@ -9,6 +9,8 @@ import ( "net/url" "os" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "github.com/buchgr/bazel-remote/v2/cache" "github.com/buchgr/bazel-remote/v2/cache/httpproxy" @@ -18,7 +20,7 @@ import ( // New creates a cache that proxies requests to Google Cloud Storage. func New(bucket string, useDefaultCredentials bool, jsonCredentialsFile string, storageMode string, - accessLogger cache.Logger, errorLogger cache.Logger, numUploaders, maxQueuedUploads int) (cache.Proxy, error) { + accessLogger cache.Logger, errorLogger cache.Logger, numUploaders, maxQueuedUploads int, otelEnabled bool) (cache.Proxy, error) { var remoteClient *http.Client var err error @@ -47,6 +49,14 @@ func New(bucket string, useDefaultCredentials bool, jsonCredentialsFile string, "credentials or a json credentials file %v", useDefaultCredentials) } + // Wrap OAuth2 client transport with OTEL instrumentation if enabled + if otelEnabled { + if remoteClient.Transport == nil { + remoteClient.Transport = http.DefaultTransport + } + remoteClient.Transport = otelhttp.NewTransport(remoteClient.Transport) + } + errorLogger.Printf("Proxying artifacts to GCS bucket '%s'.\n", bucket) baseURL := url.URL{ @@ -55,5 +65,5 @@ func New(bucket string, useDefaultCredentials bool, jsonCredentialsFile string, Path: bucket, } - return httpproxy.New(&baseURL, storageMode, remoteClient, accessLogger, errorLogger, numUploaders, maxQueuedUploads) + return httpproxy.New(&baseURL, storageMode, remoteClient, accessLogger, errorLogger, numUploaders, maxQueuedUploads, otelEnabled) } diff --git a/cache/httpproxy/BUILD.bazel b/cache/httpproxy/BUILD.bazel index b3da81320..021dac3da 100644 --- a/cache/httpproxy/BUILD.bazel +++ b/cache/httpproxy/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//utils/backendproxy:go_default_library", "@com_github_prometheus_client_golang//prometheus:go_default_library", "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", + "@io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp//:go_default_library", ], ) diff --git a/cache/httpproxy/httpproxy.go b/cache/httpproxy/httpproxy.go index 2f1acd062..09ebf7e3f 100644 --- a/cache/httpproxy/httpproxy.go +++ b/cache/httpproxy/httpproxy.go @@ -12,6 +12,8 @@ import ( "strconv" "strings" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "github.com/buchgr/bazel-remote/v2/cache" "github.com/buchgr/bazel-remote/v2/cache/disk/casblob" "github.com/buchgr/bazel-remote/v2/utils/backendproxy" @@ -98,7 +100,15 @@ func (r *remoteHTTPProxyCache) UploadFile(item backendproxy.UploadReq) { // CAS blobs) or "zstd" (which expects cas.v2 blobs). func New(baseURL *url.URL, storageMode string, remote *http.Client, accessLogger cache.Logger, errorLogger cache.Logger, - numUploaders, maxQueuedUploads int) (cache.Proxy, error) { + numUploaders, maxQueuedUploads int, otelEnabled bool) (cache.Proxy, error) { + + // Wrap HTTP client transport with OTEL instrumentation if enabled + if otelEnabled { + if remote.Transport == nil { + remote.Transport = http.DefaultTransport + } + remote.Transport = otelhttp.NewTransport(remote.Transport) + } proxy := &remoteHTTPProxyCache{ remote: remote, diff --git a/cache/httpproxy/httpproxy_test.go b/cache/httpproxy/httpproxy_test.go index ecf26c20f..9224c42fe 100644 --- a/cache/httpproxy/httpproxy_test.go +++ b/cache/httpproxy/httpproxy_test.go @@ -119,7 +119,7 @@ func TestEverything(t *testing.T) { t.Log("cas HASH:", hash) acData := []byte{1, 2, 3, 4} - proxyCache, err := New(url, "zstd", &http.Client{}, accessLogger, errorLogger, 100, 10000) + proxyCache, err := New(url, "zstd", &http.Client{}, accessLogger, errorLogger, 100, 10000, false) if err != nil { t.Fatal(err) } diff --git a/cache/s3proxy/BUILD.bazel b/cache/s3proxy/BUILD.bazel index 193f69012..9b831d5e1 100644 --- a/cache/s3proxy/BUILD.bazel +++ b/cache/s3proxy/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "@com_github_minio_minio_go_v7//pkg/credentials:go_default_library", "@com_github_prometheus_client_golang//prometheus:go_default_library", "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", + "@io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp//:go_default_library", ], ) diff --git a/cache/s3proxy/s3proxy.go b/cache/s3proxy/s3proxy.go index 5104b441b..524c70d2c 100644 --- a/cache/s3proxy/s3proxy.go +++ b/cache/s3proxy/s3proxy.go @@ -6,8 +6,11 @@ import ( "fmt" "io" "log" + "net/http" "path" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "github.com/buchgr/bazel-remote/v2/cache" "github.com/buchgr/bazel-remote/v2/cache/disk/casblob" "github.com/buchgr/bazel-remote/v2/utils/backendproxy" @@ -58,7 +61,7 @@ func New( MaxIdleConns int, storageMode string, accessLogger cache.Logger, - errorLogger cache.Logger, numUploaders, maxQueuedUploads int) cache.Proxy { + errorLogger cache.Logger, numUploaders, maxQueuedUploads int, otelEnabled bool) cache.Proxy { fmt.Println("Using S3 backend.") @@ -78,6 +81,12 @@ func New( tr.MaxIdleConns = MaxIdleConns tr.MaxIdleConnsPerHost = MaxIdleConns + // Wrap transport with OTEL instrumentation if enabled + var transport http.RoundTripper = tr + if otelEnabled { + transport = otelhttp.NewTransport(tr) + } + // Initialize minio client with credentials opts := &minio.Options{ Creds: Credentials, @@ -85,7 +94,7 @@ func New( Region: Region, Secure: secure, - Transport: tr, + Transport: transport, } minioCore, err = minio.NewCore(Endpoint, opts) if err != nil { diff --git a/config/BUILD.bazel b/config/BUILD.bazel index 5e1409d42..988cfd0dd 100644 --- a/config/BUILD.bazel +++ b/config/BUILD.bazel @@ -27,6 +27,7 @@ go_library( "@com_github_prometheus_client_golang//prometheus:go_default_library", "@com_github_urfave_cli_v2//:go_default_library", "@in_gopkg_yaml_v3//:go_default_library", + "@io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc//:go_default_library", "@org_golang_google_grpc//:go_default_library", "@org_golang_google_grpc//credentials:go_default_library", "@org_golang_google_grpc//credentials/insecure:go_default_library", diff --git a/config/config.go b/config/config.go index 05cb40172..1f08ac735 100644 --- a/config/config.go +++ b/config/config.go @@ -23,6 +23,19 @@ import ( yaml "gopkg.in/yaml.v3" ) +// OtelTracingConfig stores the OpenTelemetry tracing configuration. +type OtelTracingConfig struct { + Enabled bool `yaml:"enabled"` + ExporterEndpoint string `yaml:"exporter_endpoint"` + ServiceName string `yaml:"service_name"` + SampleRate float64 `yaml:"sample_rate"` +} + +// OtelConfig stores the OpenTelemetry configuration. +type OtelConfig struct { + Tracing *OtelTracingConfig `yaml:"tracing,omitempty"` +} + // GoogleCloudStorageConfig stores the configuration of a GCS proxy backend. type GoogleCloudStorageConfig struct { Bucket string `yaml:"bucket"` @@ -127,6 +140,7 @@ type Config struct { LogTimezone string `yaml:"log_timezone"` MaxBlobSize int64 `yaml:"max_blob_size"` MaxProxyBlobSize int64 `yaml:"max_proxy_blob_size"` + Otel *OtelConfig `yaml:"otel,omitempty"` // Fields that are created by combinations of the flags above. ProxyBackend cache.Proxy @@ -173,6 +187,7 @@ func newFromArgs(dir string, maxSize int, storageMode string, zstdImplementation ldap *LDAPConfig, s3 *S3CloudStorageConfig, azblob *AzBlobStorageConfig, + otel *OtelConfig, disableHTTPACValidation bool, disableGRPCACDepsCheck bool, enableACKeyInstanceMangling bool, @@ -210,6 +225,7 @@ func newFromArgs(dir string, maxSize int, storageMode string, zstdImplementation HTTPBackend: hc, GRPCBackend: grpcb, LDAP: ldap, + Otel: otel, IdleTimeout: idleTimeout, DisableHTTPACValidation: disableHTTPACValidation, DisableGRPCACDepsCheck: disableGRPCACDepsCheck, @@ -500,6 +516,36 @@ func validateConfig(c *Config) error { return errors.New("'log_timezone' must be set to either \"UTC\", \"local\" or \"none\"") } + // Validate OpenTelemetry tracing configuration + if c.Otel != nil && c.Otel.Tracing != nil && c.Otel.Tracing.Enabled { + if c.Otel.Tracing.ExporterEndpoint == "" { + // Check standard OTEL env vars + if endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); endpoint != "" { + c.Otel.Tracing.ExporterEndpoint = endpoint + } else if endpoint := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"); endpoint != "" { + c.Otel.Tracing.ExporterEndpoint = endpoint + } else { + return errors.New("when 'otel.tracing.enabled' is true, either 'otel.tracing.exporter_endpoint' must be set or OTEL_EXPORTER_OTLP_ENDPOINT env var must be defined") + } + } + + if c.Otel.Tracing.ServiceName == "" { + if name := os.Getenv("OTEL_SERVICE_NAME"); name != "" { + c.Otel.Tracing.ServiceName = name + } else { + c.Otel.Tracing.ServiceName = "bazel-remote" + } + } + + if c.Otel.Tracing.SampleRate == 0 { + c.Otel.Tracing.SampleRate = 1.0 // Default to 100% sampling + } + + if c.Otel.Tracing.SampleRate < 0.0 || c.Otel.Tracing.SampleRate > 1.0 { + return errors.New("'otel.tracing.sample_rate' must be between 0.0 and 1.0") + } + } + return nil } @@ -643,6 +689,18 @@ func get(ctx *cli.Context) (*Config, error) { } } + var otel *OtelConfig + if ctx.Bool("otel.tracing.enabled") { + otel = &OtelConfig{ + Tracing: &OtelTracingConfig{ + Enabled: ctx.Bool("otel.tracing.enabled"), + ExporterEndpoint: ctx.String("otel.tracing.exporter_endpoint"), + ServiceName: ctx.String("otel.tracing.service_name"), + SampleRate: ctx.Float64("otel.tracing.sample_rate"), + }, + } + } + return newFromArgs( ctx.String("dir"), ctx.Int("max_size"), @@ -666,6 +724,7 @@ func get(ctx *cli.Context) (*Config, error) { ldap, s3, azblob, + otel, ctx.Bool("disable_http_ac_validation"), ctx.Bool("disable_grpc_ac_deps_check"), ctx.Bool("enable_ac_key_instance_mangling"), diff --git a/config/config_test.go b/config/config_test.go index 9fc4cce60..9f2cd46b7 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "math" "net/url" + "os" "reflect" "regexp" "strings" @@ -60,6 +61,7 @@ log_timezone: local MetricsDurationBuckets: []float64{.5, 1, 2.5, 5, 10, 20, 40, 80, 160, 320}, AccessLogLevel: "none", LogTimezone: "local", + Otel: nil, } if !reflect.DeepEqual(config, expectedConfig) { @@ -103,6 +105,7 @@ gcs_proxy: MetricsDurationBuckets: []float64{.5, 1, 2.5, 5, 10, 20, 40, 80, 160, 320}, AccessLogLevel: "all", LogTimezone: "UTC", + Otel: nil, } if !cmp.Equal(config, expectedConfig) { @@ -147,6 +150,7 @@ http_proxy: MetricsDurationBuckets: []float64{.5, 1, 2.5, 5, 10, 20, 40, 80, 160, 320}, AccessLogLevel: "all", LogTimezone: "UTC", + Otel: nil, } if !cmp.Equal(config, expectedConfig) { @@ -544,3 +548,239 @@ func TestSocketPathMissing(t *testing.T) { t.Fatal("Expected the error message to mention the missing 'http_address' key/flag") } } + +func TestOtelConfigValidation(t *testing.T) { + tests := []struct { + name string + yaml string + envVars map[string]string + wantErr bool + errContains string + checkFunc func(*testing.T, *Config) + }{ + { + name: "valid config with all fields", + yaml: `dir: /tmp/test +max_size: 100 +otel: + tracing: + enabled: true + exporter_endpoint: localhost:4317 + service_name: test-service + sample_rate: 0.5 +`, + wantErr: false, + checkFunc: func(t *testing.T, c *Config) { + if c.Otel == nil || !c.Otel.Tracing.Enabled { + t.Error("Expected OTEL to be enabled") + } + if c.Otel.Tracing.ExporterEndpoint != "localhost:4317" { + t.Errorf("Expected endpoint localhost:4317, got %s", c.Otel.Tracing.ExporterEndpoint) + } + if c.Otel.Tracing.ServiceName != "test-service" { + t.Errorf("Expected service name test-service, got %s", c.Otel.Tracing.ServiceName) + } + if c.Otel.Tracing.SampleRate != 0.5 { + t.Errorf("Expected sample rate 0.5, got %f", c.Otel.Tracing.SampleRate) + } + }, + }, + { + name: "valid config with defaults", + yaml: `dir: /tmp/test +max_size: 100 +otel: + tracing: + enabled: true + exporter_endpoint: localhost:4317 +`, + wantErr: false, + checkFunc: func(t *testing.T, c *Config) { + if c.Otel == nil || !c.Otel.Tracing.Enabled { + t.Error("Expected OTEL to be enabled") + } + if c.Otel.Tracing.ServiceName != "bazel-remote" { + t.Errorf("Expected default service name bazel-remote, got %s", c.Otel.Tracing.ServiceName) + } + if c.Otel.Tracing.SampleRate != 1.0 { + t.Errorf("Expected default sample rate 1.0, got %f", c.Otel.Tracing.SampleRate) + } + }, + }, + { + name: "missing endpoint with env var", + yaml: `dir: /tmp/test +max_size: 100 +otel: + tracing: + enabled: true +`, + envVars: map[string]string{ + "OTEL_EXPORTER_OTLP_ENDPOINT": "env-endpoint:4317", + }, + wantErr: false, + checkFunc: func(t *testing.T, c *Config) { + if c.Otel.Tracing.ExporterEndpoint != "env-endpoint:4317" { + t.Errorf("Expected endpoint from env var, got %s", c.Otel.Tracing.ExporterEndpoint) + } + }, + }, + { + name: "missing endpoint without env var", + yaml: `dir: /tmp/test +max_size: 100 +otel: + tracing: + enabled: true +`, + wantErr: true, + errContains: "exporter_endpoint", + }, + { + name: "invalid sample rate negative", + yaml: `dir: /tmp/test +max_size: 100 +otel: + tracing: + enabled: true + exporter_endpoint: localhost:4317 + sample_rate: -0.1 +`, + wantErr: true, + errContains: "sample_rate", + }, + { + name: "invalid sample rate greater than 1", + yaml: `dir: /tmp/test +max_size: 100 +otel: + tracing: + enabled: true + exporter_endpoint: localhost:4317 + sample_rate: 1.5 +`, + wantErr: true, + errContains: "sample_rate", + }, + { + name: "disabled no validation", + yaml: `dir: /tmp/test +max_size: 100 +otel: + tracing: + enabled: false +`, + wantErr: false, + }, + { + name: "no otel config", + yaml: `dir: /tmp/test +max_size: 100 +`, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set up environment variables + cleanup := make([]func(), 0) + for k, v := range tt.envVars { + oldVal := os.Getenv(k) + os.Setenv(k, v) + cleanup = append(cleanup, func() { + if oldVal != "" { + os.Setenv(k, oldVal) + } else { + os.Unsetenv(k) + } + }) + } + defer func() { + for _, fn := range cleanup { + fn() + } + }() + + // Clear env vars that might interfere + if len(tt.envVars) == 0 { + oldEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + oldTracesEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + oldServiceName := os.Getenv("OTEL_SERVICE_NAME") + os.Unsetenv("OTEL_EXPORTER_OTLP_ENDPOINT") + os.Unsetenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + os.Unsetenv("OTEL_SERVICE_NAME") + defer func() { + if oldEndpoint != "" { + os.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", oldEndpoint) + } + if oldTracesEndpoint != "" { + os.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", oldTracesEndpoint) + } + if oldServiceName != "" { + os.Setenv("OTEL_SERVICE_NAME", oldServiceName) + } + }() + } + + config, err := NewFromYaml([]byte(tt.yaml)) + if (err != nil) != tt.wantErr { + t.Errorf("NewFromYaml() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if tt.wantErr && tt.errContains != "" { + if err == nil || !strings.Contains(err.Error(), tt.errContains) { + t.Errorf("NewFromYaml() error = %v, want error containing %q", err, tt.errContains) + } + return + } + + if !tt.wantErr && tt.checkFunc != nil { + tt.checkFunc(t, config) + } + }) + } +} + +func TestOtelDisabledByDefault(t *testing.T) { + // Test that OTEL is disabled when no otel config is specified + yaml := `dir: /tmp/test +max_size: 100 +` + config, err := NewFromYaml([]byte(yaml)) + if err != nil { + t.Fatalf("NewFromYaml() failed: %v", err) + } + + // Otel should be nil when not specified + if config.Otel != nil { + t.Errorf("Expected Otel to be nil when not specified, got %+v", config.Otel) + } +} + +func TestOtelDisabledWhenEnabledIsFalse(t *testing.T) { + // Test that OTEL is disabled when explicitly set to false + yaml := `dir: /tmp/test +max_size: 100 +otel: + tracing: + enabled: false + exporter_endpoint: localhost:4317 +` + config, err := NewFromYaml([]byte(yaml)) + if err != nil { + t.Fatalf("NewFromYaml() failed: %v", err) + } + + // Otel should exist but be disabled + if config.Otel == nil { + t.Fatal("Expected Otel config to exist") + } + if config.Otel.Tracing == nil { + t.Fatal("Expected Otel.Tracing config to exist") + } + if config.Otel.Tracing.Enabled { + t.Error("Expected Otel.Tracing.Enabled to be false") + } +} diff --git a/config/proxy.go b/config/proxy.go index 1ec0bba69..0f7a3656f 100644 --- a/config/proxy.go +++ b/config/proxy.go @@ -9,6 +9,8 @@ import ( "net/http" "os" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "github.com/buchgr/bazel-remote/v2/cache/azblobproxy" "github.com/buchgr/bazel-remote/v2/cache/gcsproxy" "github.com/buchgr/bazel-remote/v2/cache/grpcproxy" @@ -49,10 +51,12 @@ func getTLSConfig(certFile, keyFile, caFile string) (*tls.Config, error) { } func (c *Config) setProxy() error { + otelEnabled := c.Otel != nil && c.Otel.Tracing != nil && c.Otel.Tracing.Enabled + if c.GoogleCloudStorage != nil { proxyCache, err := gcsproxy.New(c.GoogleCloudStorage.Bucket, c.GoogleCloudStorage.UseDefaultCredentials, c.GoogleCloudStorage.JSONCredentialsFile, - c.StorageMode, c.AccessLogger, c.ErrorLogger, c.NumUploaders, c.MaxQueuedUploads) + c.StorageMode, c.AccessLogger, c.ErrorLogger, c.NumUploaders, c.MaxQueuedUploads, otelEnabled) if err != nil { return err } @@ -63,6 +67,15 @@ func (c *Config) setProxy() error { if c.GRPCBackend != nil { var opts []grpc.DialOption + + // Add OTEL client interceptors if enabled + if otelEnabled { + opts = append(opts, + grpc.WithChainStreamInterceptor(otelgrpc.StreamClientInterceptor()), + grpc.WithChainUnaryInterceptor(otelgrpc.UnaryClientInterceptor()), + ) + } + if c.GRPCBackend.BaseURL.Scheme == "grpcs" { config, err := getTLSConfig(c.GRPCBackend.CertFile, c.GRPCBackend.KeyFile, c.GRPCBackend.CaFile) if err != nil { @@ -122,7 +135,7 @@ func (c *Config) setProxy() error { } proxyCache, err := httpproxy.New(c.HTTPBackend.BaseURL, c.StorageMode, - httpClient, c.AccessLogger, c.ErrorLogger, c.NumUploaders, c.MaxQueuedUploads) + httpClient, c.AccessLogger, c.ErrorLogger, c.NumUploaders, c.MaxQueuedUploads, otelEnabled) if err != nil { return err } @@ -151,7 +164,7 @@ func (c *Config) setProxy() error { c.S3CloudStorage.UpdateTimestamps, c.S3CloudStorage.Region, c.S3CloudStorage.MaxIdleConns, - c.StorageMode, c.AccessLogger, c.ErrorLogger, c.NumUploaders, c.MaxQueuedUploads) + c.StorageMode, c.AccessLogger, c.ErrorLogger, c.NumUploaders, c.MaxQueuedUploads, otelEnabled) return nil } diff --git a/go.mod b/go.mod index ec352bd0a..1a377ec11 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/buchgr/bazel-remote/v2 require ( - cloud.google.com/go/longrunning v0.6.7 + cloud.google.com/go/longrunning v0.6.4 github.com/abbot/go-http-auth v0.4.1-0.20220112235402-e1cee1c72f2f github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/djherbis/atime v1.1.0 @@ -19,8 +19,8 @@ require ( golang.org/x/oauth2 v0.31.0 golang.org/x/sync v0.17.0 golang.org/x/sys v0.36.0 // indirect - google.golang.org/grpc v1.75.1 - google.golang.org/protobuf v1.36.9 + google.golang.org/grpc v1.75.0 + google.golang.org/protobuf v1.36.10 gopkg.in/yaml.v3 v3.0.1 ) @@ -31,8 +31,13 @@ require ( github.com/go-ldap/ldap/v3 v3.4.11 github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877 github.com/valyala/gozstd v1.23.2 - google.golang.org/genproto/googleapis/api v0.0.0-20250922171735-9219d122eba9 - google.golang.org/genproto/googleapis/bytestream v0.0.0-20250922171735-9219d122eba9 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 + go.opentelemetry.io/otel/sdk v1.38.0 + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 + google.golang.org/genproto/googleapis/bytestream v0.0.0-20251124214823-79d6a2a48846 google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9 ) @@ -43,13 +48,18 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect github.com/aws/aws-sdk-go v1.44.256 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golang/snappy v1.0.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/minio/crc64nvme v1.1.1 // indirect @@ -65,6 +75,11 @@ require ( github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect github.com/tinylib/msgp v1.4.0 // indirect github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/crypto v0.42.0 // indirect golang.org/x/net v0.44.0 // indirect diff --git a/go.sum b/go.sum index d0e69242a..1e38434af 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= -cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= +cloud.google.com/go/longrunning v0.6.4 h1:3tyw9rO3E2XVXzSApn1gyEEnH2K9SynNQjMlBi3uHLg= +cloud.google.com/go/longrunning v0.6.4/go.mod h1:ttZpLCe6e7EXvn9OxpBRx7kZEB0efv8yBO6YnVMfhJs= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 h1:5YTBM8QDVIBN3sxBil89WfdAAqDZbyJTgh688DSxX5w= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.12.0 h1:wL5IEG5zb7BVv1Kv0Xm92orq+5hB5Nipn3B5tn4Rqfk= @@ -28,6 +28,8 @@ github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVW github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= @@ -39,12 +41,15 @@ github.com/djherbis/atime v1.1.0 h1:rgwVbP/5by8BvvjBNrbh64Qz33idKT3pSnMSJsxhi0g= github.com/djherbis/atime v1.1.0/go.mod h1:28OF6Y8s3NQWwacXc5eZTsEsiMzp7LF8MbXE+XJPdBE= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo= github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ldap/ldap/v3 v3.4.11 h1:4k0Yxweg+a3OyBLjdYn5OKglv18JNvfDykSoI8bW0gU= github.com/go-ldap/ldap/v3 v3.4.11/go.mod h1:bY7t0FLK8OAVpp/vV6sSlpz3EQDGcQwc8pF0ujLgKvM= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -63,6 +68,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= @@ -122,8 +129,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -153,16 +160,26 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= @@ -224,16 +241,16 @@ golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20250922171735-9219d122eba9 h1:jm6v6kMRpTYKxBRrDkYAitNJegUeO1Mf3Kt80obv0gg= -google.golang.org/genproto/googleapis/api v0.0.0-20250922171735-9219d122eba9/go.mod h1:LmwNphe5Afor5V3R5BppOULHOnt2mCIf+NxMd4XiygE= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20250922171735-9219d122eba9 h1:kd3gJFNX/jKKVdSvHximKXHVeipNPbrgyBSxfYAR2ew= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20250922171735-9219d122eba9/go.mod h1:YUQUKndxDbAanQC0ln4pZ3Sis3N5sqgDte2XQqufkJc= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20251124214823-79d6a2a48846 h1:7FlucM2tFADtEDnIlDrR12KdRqV48B1GSTU1U6uKSiY= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20251124214823-79d6a2a48846/go.mod h1:G3Q0qS3k/oFEmVMddPsSYcFnm2+Mq2XRmxujrtu5hr0= google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9 h1:V1jCN2HBa8sySkR5vLcCSqJSTMv093Rw9EJefhQGP7M= google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9/go.mod h1:HSkG/KdJWusxU1F6CNrwNDjBMgisKxGnc5dAZfT0mjQ= -google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= -google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= -google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/main.go b/main.go index f2280590d..ee035a53a 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ import ( "github.com/buchgr/bazel-remote/v2/config" "github.com/buchgr/bazel-remote/v2/ldap" + "github.com/buchgr/bazel-remote/v2/otel" "github.com/buchgr/bazel-remote/v2/server" "github.com/buchgr/bazel-remote/v2/utils/flags" "github.com/buchgr/bazel-remote/v2/utils/idle" @@ -31,6 +32,9 @@ import ( middlewarestd "github.com/slok/go-http-metrics/middleware/std" "github.com/urfave/cli/v2" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -96,6 +100,27 @@ func run(ctx *cli.Context) error { rlimit.Raise() + // Initialize OpenTelemetry if enabled + var otelProvider *otel.TracerProvider + if c.Otel != nil && c.Otel.Tracing != nil && c.Otel.Tracing.Enabled { + otelProvider, err = otel.InitTracer( + c.Otel.Tracing.ExporterEndpoint, + c.Otel.Tracing.ServiceName, + c.Otel.Tracing.SampleRate, + ) + if err != nil { + log.Printf("WARNING: Failed to initialize OpenTelemetry: %v", err) + // Continue without OTEL (graceful degradation) + } else { + defer func() { + shutdownCtx := context.Background() + if err := otelProvider.Shutdown(shutdownCtx); err != nil { + log.Printf("Error shutting down OTEL tracer: %v", err) + } + }() + } + } + grpcSem := semaphore.NewWeighted(1) var grpcServer *grpc.Server @@ -281,6 +306,18 @@ func startHttpServer(c *config.Config, httpServer **http.Server, }) } + // Wrap with OTEL tracing if enabled + if c.Otel != nil && c.Otel.Tracing != nil && c.Otel.Tracing.Enabled { + cacheHandler = otelhttp.NewHandler( + http.HandlerFunc(cacheHandler), + "cache", + otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string { + return fmt.Sprintf("%s %s", r.Method, r.URL.Path) + }), + ).ServeHTTP + log.Println("HTTP OpenTelemetry tracing: enabled") + } + var statusHandler http.HandlerFunc = h.StatusPageHandler if !c.AllowUnauthenticatedReads { @@ -293,6 +330,14 @@ func startHttpServer(c *config.Config, httpServer **http.Server, } } + // Wrap status handler with OTEL tracing if enabled + if c.Otel != nil && c.Otel.Tracing != nil && c.Otel.Tracing.Enabled { + statusHandler = otelhttp.NewHandler( + http.HandlerFunc(statusHandler), + "status", + ).ServeHTTP + } + if c.EnableEndpointMetrics { log.Println("Endpoint metrics: enabled") @@ -393,6 +438,15 @@ func startGrpcServer(c *config.Config, grpcServer **grpc.Server, streamInterceptors := []grpc.StreamServerInterceptor{} unaryInterceptors := []grpc.UnaryServerInterceptor{} + // Add OTEL interceptors first for complete trace coverage + if c.Otel != nil && c.Otel.Tracing != nil && c.Otel.Tracing.Enabled { + streamInterceptors = append(streamInterceptors, + otelgrpc.StreamServerInterceptor()) + unaryInterceptors = append(unaryInterceptors, + otelgrpc.UnaryServerInterceptor()) + log.Println("gRPC OpenTelemetry tracing: enabled") + } + if c.EnableEndpointMetrics { streamInterceptors = append(streamInterceptors, grpc_prometheus.StreamServerInterceptor) unaryInterceptors = append(unaryInterceptors, grpc_prometheus.UnaryServerInterceptor) diff --git a/otel/BUILD.bazel b/otel/BUILD.bazel new file mode 100644 index 000000000..3e76b8149 --- /dev/null +++ b/otel/BUILD.bazel @@ -0,0 +1,16 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["otel.go"], + importpath = "github.com/buchgr/bazel-remote/v2/otel", + visibility = ["//visibility:public"], + deps = [ + "@io_opentelemetry_go_otel//:go_default_library", + "@io_opentelemetry_go_otel//propagation:go_default_library", + "@io_opentelemetry_go_otel//semconv/v1.37.0:go_default_library", + "@io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracegrpc//:go_default_library", + "@io_opentelemetry_go_otel_sdk//resource:go_default_library", + "@io_opentelemetry_go_otel_sdk//trace:go_default_library", + ], +) diff --git a/otel/otel.go b/otel/otel.go new file mode 100644 index 000000000..d77eb71dd --- /dev/null +++ b/otel/otel.go @@ -0,0 +1,77 @@ +package otel + +import ( + "context" + "fmt" + "log" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.37.0" +) + +// TracerProvider wraps the OpenTelemetry tracer provider for lifecycle management +type TracerProvider struct { + tp *sdktrace.TracerProvider +} + +// InitTracer initializes the OpenTelemetry tracer provider with OTLP gRPC exporter +func InitTracer(endpoint, serviceName string, sampleRate float64) (*TracerProvider, error) { + ctx := context.Background() + + // Create OTLP gRPC exporter + exporter, err := otlptracegrpc.New(ctx, + otlptracegrpc.WithEndpoint(endpoint), + // TLS is usually not typically required if running in a k8s environment + otlptracegrpc.WithInsecure(), + ) + if err != nil { + return nil, fmt.Errorf("failed to create OTLP trace exporter: %w", err) + } + + // Create resource with service information + res, err := resource.New(ctx, + resource.WithAttributes( + semconv.ServiceNameKey.String(serviceName), + ), + ) + if err != nil { + return nil, fmt.Errorf("failed to create resource: %w", err) + } + + // Create tracer provider with sampling + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.TraceIDRatioBased(sampleRate)), + ) + + // Set as global tracer provider + otel.SetTracerProvider(tp) + + // Set global propagator for W3C Trace Context + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + + log.Printf("OpenTelemetry initialized: endpoint=%s service=%s sample_rate=%.2f", + endpoint, serviceName, sampleRate) + + return &TracerProvider{tp: tp}, nil +} + +// Shutdown gracefully shuts down the tracer provider +func (tp *TracerProvider) Shutdown(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + if err := tp.tp.Shutdown(ctx); err != nil { + return fmt.Errorf("error shutting down tracer provider: %w", err) + } + return nil +} diff --git a/utils/flags/flags.go b/utils/flags/flags.go index 38018c576..db30e055d 100644 --- a/utils/flags/flags.go +++ b/utils/flags/flags.go @@ -527,5 +527,31 @@ func GetCliFlags() []cli.Flag { DefaultText: "UTC, ie use UTC timezone", EnvVars: []string{"BAZEL_REMOTE_LOG_TIMEZONE"}, }, + &cli.BoolFlag{ + Name: "otel.tracing.enabled", + Usage: "Whether to enable OpenTelemetry distributed tracing.", + DefaultText: "false, ie disable OTEL tracing", + EnvVars: []string{"BAZEL_REMOTE_OTEL_TRACING_ENABLED"}, + }, + &cli.StringFlag{ + Name: "otel.tracing.exporter_endpoint", + Usage: "The OTLP exporter endpoint for OpenTelemetry traces. Can also be set via OTEL_EXPORTER_OTLP_ENDPOINT env var.", + DefaultText: "empty, will use OTEL_EXPORTER_OTLP_ENDPOINT env var", + EnvVars: []string{"BAZEL_REMOTE_OTEL_TRACING_EXPORTER_ENDPOINT"}, + }, + &cli.StringFlag{ + Name: "otel.tracing.service_name", + Usage: "The service name for OpenTelemetry traces. Can also be set via OTEL_SERVICE_NAME env var.", + Value: "bazel-remote", + DefaultText: "bazel-remote", + EnvVars: []string{"BAZEL_REMOTE_OTEL_TRACING_SERVICE_NAME"}, + }, + &cli.Float64Flag{ + Name: "otel.tracing.sample_rate", + Usage: "The sampling rate for OpenTelemetry traces (0.0-1.0). 1.0 means 100% sampling.", + Value: 1.0, + DefaultText: "1.0 (100% sampling)", + EnvVars: []string{"BAZEL_REMOTE_OTEL_TRACING_SAMPLE_RATE"}, + }, } }