diff --git a/app/vtselect/extrafilters/extrafilters.go b/app/vtselect/extrafilters/extrafilters.go new file mode 100644 index 000000000..316b5cc30 --- /dev/null +++ b/app/vtselect/extrafilters/extrafilters.go @@ -0,0 +1,129 @@ +// Package extrafilters parses the extra_filters and extra_stream_filters query args. +// +// The args are shared by the native LogsQL endpoints and the Jaeger and Tempo APIs, +// so the parsing lives here rather than in one of them. +package extrafilters + +import ( + "fmt" + "regexp" + "strings" + + "github.com/valyala/fastjson" + + "github.com/VictoriaMetrics/VictoriaLogs/lib/logstorage" +) + +// ParseExtraFilters parses the extra_filters query arg. +func ParseExtraFilters(s string) (*logstorage.Filter, error) { + if s == "" { + return nil, nil + } + if !strings.HasPrefix(s, `{"`) { + return logstorage.ParseFilter(s) + } + + // Extra filters in the form {"field":"value",...}. + kvs, err := parseExtraFiltersJSON(s) + if err != nil { + return nil, err + } + + filters := make([]string, len(kvs)) + for i, kv := range kvs { + if len(kv.values) == 1 { + filters[i] = fmt.Sprintf("%q:=%q", kv.key, kv.values[0]) + } else { + orValues := make([]string, len(kv.values)) + for j, v := range kv.values { + orValues[j] = fmt.Sprintf("%q", v) + } + filters[i] = fmt.Sprintf("%q:in(%s)", kv.key, strings.Join(orValues, ",")) + } + } + s = strings.Join(filters, " ") + return logstorage.ParseFilter(s) +} + +// ParseExtraStreamFilters parses the extra_stream_filters query arg. +func ParseExtraStreamFilters(s string) (*logstorage.Filter, error) { + if s == "" { + return nil, nil + } + if !strings.HasPrefix(s, `{"`) { + return logstorage.ParseFilter(s) + } + + // Extra stream filters in the form {"field":"value",...}. + kvs, err := parseExtraFiltersJSON(s) + if err != nil { + return nil, err + } + + filters := make([]string, len(kvs)) + for i, kv := range kvs { + if len(kv.values) == 1 { + filters[i] = fmt.Sprintf("%q=%q", kv.key, kv.values[0]) + } else { + orValues := make([]string, len(kv.values)) + for j, v := range kv.values { + orValues[j] = regexp.QuoteMeta(v) + } + filters[i] = fmt.Sprintf("%q=~%q", kv.key, strings.Join(orValues, "|")) + } + } + s = "{" + strings.Join(filters, ",") + "}" + return logstorage.ParseFilter(s) +} + +type extraFilter struct { + key string + values []string +} + +func parseExtraFiltersJSON(s string) ([]extraFilter, error) { + v, err := fastjson.Parse(s) + if err != nil { + return nil, err + } + o := v.GetObject() + + var errOuter error + var filters []extraFilter + o.Visit(func(k []byte, v *fastjson.Value) { + if errOuter != nil { + return + } + switch v.Type() { + case fastjson.TypeString: + filters = append(filters, extraFilter{ + key: string(k), + values: []string{string(v.GetStringBytes())}, + }) + case fastjson.TypeArray: + a := v.GetArray() + if len(a) == 0 { + return + } + orValues := make([]string, len(a)) + for i, av := range a { + ov, err := av.StringBytes() + if err != nil { + errOuter = fmt.Errorf("cannot obtain string item at the array for key %q; item: %s", k, av) + return + } + orValues[i] = string(ov) + } + filters = append(filters, extraFilter{ + key: string(k), + values: orValues, + }) + default: + errOuter = fmt.Errorf("unexpected type of value for key %q: %s; value: %s", k, v.Type(), v) + } + }) + if errOuter != nil { + return nil, errOuter + } + return filters, nil +} diff --git a/app/vtselect/logsql/logsql_test.go b/app/vtselect/extrafilters/extrafilters_test.go similarity index 92% rename from app/vtselect/logsql/logsql_test.go rename to app/vtselect/extrafilters/extrafilters_test.go index 0cb30c62a..b68f45cc1 100644 --- a/app/vtselect/logsql/logsql_test.go +++ b/app/vtselect/extrafilters/extrafilters_test.go @@ -1,4 +1,4 @@ -package logsql +package extrafilters import ( "testing" @@ -8,7 +8,7 @@ func TestParseExtraFilters_Success(t *testing.T) { f := func(s, resultExpected string) { t.Helper() - f, err := parseExtraFilters(s) + f, err := ParseExtraFilters(s) if err != nil { t.Fatalf("unexpected error in parseExtraFilters: %s", err) } @@ -35,7 +35,7 @@ func TestParseExtraFilters_Failure(t *testing.T) { f := func(s string) { t.Helper() - _, err := parseExtraFilters(s) + _, err := ParseExtraFilters(s) if err == nil { t.Fatalf("expecting non-nil error") } @@ -57,7 +57,7 @@ func TestParseExtraStreamFilters_Success(t *testing.T) { f := func(s, resultExpected string) { t.Helper() - f, err := parseExtraStreamFilters(s) + f, err := ParseExtraStreamFilters(s) if err != nil { t.Fatalf("unexpected error in parseExtraStreamFilters: %s", err) } @@ -84,7 +84,7 @@ func TestParseExtraStreamFilters_Failure(t *testing.T) { f := func(s string) { t.Helper() - _, err := parseExtraStreamFilters(s) + _, err := ParseExtraStreamFilters(s) if err == nil { t.Fatalf("expecting non-nil error") } diff --git a/app/vtselect/logsql/logsql.go b/app/vtselect/logsql/logsql.go index 91bff8f30..09687b6d5 100644 --- a/app/vtselect/logsql/logsql.go +++ b/app/vtselect/logsql/logsql.go @@ -8,7 +8,6 @@ import ( "io" "math" "net/http" - "regexp" "slices" "sort" "strconv" @@ -26,9 +25,9 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil" "github.com/VictoriaMetrics/metrics" - "github.com/valyala/fastjson" "github.com/valyala/quicktemplate" + "github.com/VictoriaMetrics/VictoriaTraces/app/vtselect/extrafilters" "github.com/VictoriaMetrics/VictoriaTraces/app/vtstorage" ) @@ -1553,7 +1552,7 @@ func parseCommonArgsWithConfig(r *http.Request, skipMaxRangeCheck bool) (*common // Parse optional extra_filters for _, extraFiltersStr := range r.Form["extra_filters"] { - extraFilters, err := parseExtraFilters(extraFiltersStr) + extraFilters, err := extrafilters.ParseExtraFilters(extraFiltersStr) if err != nil { return nil, err } @@ -1562,7 +1561,7 @@ func parseCommonArgsWithConfig(r *http.Request, skipMaxRangeCheck bool) (*common // Parse optional extra_stream_filters for _, extraStreamFiltersStr := range r.Form["extra_stream_filters"] { - extraStreamFilters, err := parseExtraStreamFilters(extraStreamFiltersStr) + extraStreamFilters, err := extrafilters.ParseExtraStreamFilters(extraStreamFiltersStr) if err != nil { return nil, err } @@ -1652,118 +1651,6 @@ func getTimeNsec(r *http.Request, argName string) (int64, bool, error) { return nsecs, true, nil } -func parseExtraFilters(s string) (*logstorage.Filter, error) { - if s == "" { - return nil, nil - } - if !strings.HasPrefix(s, `{"`) { - return logstorage.ParseFilter(s) - } - - // Extra filters in the form {"field":"value",...}. - kvs, err := parseExtraFiltersJSON(s) - if err != nil { - return nil, err - } - - filters := make([]string, len(kvs)) - for i, kv := range kvs { - if len(kv.values) == 1 { - filters[i] = fmt.Sprintf("%q:=%q", kv.key, kv.values[0]) - } else { - orValues := make([]string, len(kv.values)) - for j, v := range kv.values { - orValues[j] = fmt.Sprintf("%q", v) - } - filters[i] = fmt.Sprintf("%q:in(%s)", kv.key, strings.Join(orValues, ",")) - } - } - s = strings.Join(filters, " ") - return logstorage.ParseFilter(s) -} - -func parseExtraStreamFilters(s string) (*logstorage.Filter, error) { - if s == "" { - return nil, nil - } - if !strings.HasPrefix(s, `{"`) { - return logstorage.ParseFilter(s) - } - - // Extra stream filters in the form {"field":"value",...}. - kvs, err := parseExtraFiltersJSON(s) - if err != nil { - return nil, err - } - - filters := make([]string, len(kvs)) - for i, kv := range kvs { - if len(kv.values) == 1 { - filters[i] = fmt.Sprintf("%q=%q", kv.key, kv.values[0]) - } else { - orValues := make([]string, len(kv.values)) - for j, v := range kv.values { - orValues[j] = regexp.QuoteMeta(v) - } - filters[i] = fmt.Sprintf("%q=~%q", kv.key, strings.Join(orValues, "|")) - } - } - s = "{" + strings.Join(filters, ",") + "}" - return logstorage.ParseFilter(s) -} - -type extraFilter struct { - key string - values []string -} - -func parseExtraFiltersJSON(s string) ([]extraFilter, error) { - v, err := fastjson.Parse(s) - if err != nil { - return nil, err - } - o := v.GetObject() - - var errOuter error - var filters []extraFilter - o.Visit(func(k []byte, v *fastjson.Value) { - if errOuter != nil { - return - } - switch v.Type() { - case fastjson.TypeString: - filters = append(filters, extraFilter{ - key: string(k), - values: []string{string(v.GetStringBytes())}, - }) - case fastjson.TypeArray: - a := v.GetArray() - if len(a) == 0 { - return - } - orValues := make([]string, len(a)) - for i, av := range a { - ov, err := av.StringBytes() - if err != nil { - errOuter = fmt.Errorf("cannot obtain string item at the array for key %q; item: %s", k, av) - return - } - orValues[i] = string(ov) - } - filters = append(filters, extraFilter{ - key: string(k), - values: orValues, - }) - default: - errOuter = fmt.Errorf("unexpected type of value for key %q: %s; value: %s", k, v.Type(), v) - } - }) - if errOuter != nil { - return nil, errOuter - } - return filters, nil -} - func getPositiveInt(r *http.Request, argName string) (int, error) { n, err := httputil.GetInt(r, argName) if err != nil { diff --git a/app/vtselect/traces/tracecommon/tracecommon.go b/app/vtselect/traces/tracecommon/tracecommon.go index db0a85b3c..5f55bf50d 100644 --- a/app/vtselect/traces/tracecommon/tracecommon.go +++ b/app/vtselect/traces/tracecommon/tracecommon.go @@ -12,6 +12,7 @@ import ( "github.com/VictoriaMetrics/VictoriaLogs/lib/logstorage" + "github.com/VictoriaMetrics/VictoriaTraces/app/vtselect/extrafilters" "github.com/VictoriaMetrics/VictoriaTraces/app/vtstorage" ) @@ -57,11 +58,21 @@ type CommonParams struct { // Optional list of log fields or log field prefixes ending with *, which must be hidden during query execution. HiddenFieldsFilters []string + // Optional filters from the extra_filters and extra_stream_filters query args. + // They are applied to Query in NewQueryContext, so every traces API respects them. + ExtraFilters []*logstorage.Filter + // qs contains execution statistics for the Query. qs logstorage.QueryStats } func (cp *CommonParams) NewQueryContext(ctx context.Context) *logstorage.QueryContext { + // Every traces API builds its own Query and then calls this, so applying the extra filters + // here covers all of them. AddExtraFilters ANDs the filters in, so a repeated call is harmless. + for _, f := range cp.ExtraFilters { + cp.Query.AddExtraFilters(f) + } + return logstorage.NewQueryContext(ctx, &cp.qs, cp.TenantIDs, cp.Query, cp.AllowPartialResponse, cp.HiddenFieldsFilters) } @@ -82,14 +93,56 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) { return nil, err } + extraFilters, err := getExtraFilters(r) + if err != nil { + return nil, err + } + cp := &CommonParams{ TenantIDs: tenantIDs, HiddenFieldsFilters: hiddenFieldsFilters, + ExtraFilters: extraFilters, } return cp, nil } +// getExtraFilters parses the extra_filters and extra_stream_filters query args. +// +// vmauth injects these args in order to restrict a user to its own data, +// so dropping them would show that user the data of others. +// See https://github.com/VictoriaMetrics/VictoriaTraces/issues/178 +func getExtraFilters(r *http.Request) ([]*logstorage.Filter, error) { + // The args may repeat, so they are read from r.Form rather than via r.FormValue. + // r.Form stays nil until the form is parsed. + if err := r.ParseForm(); err != nil { + return nil, fmt.Errorf("cannot parse request args: %w", err) + } + + var fs []*logstorage.Filter + + for _, s := range r.Form["extra_filters"] { + f, err := extrafilters.ParseExtraFilters(s) + if err != nil { + return nil, fmt.Errorf("cannot parse extra_filters=%q: %w", s, err) + } + if f != nil { + fs = append(fs, f) + } + } + for _, s := range r.Form["extra_stream_filters"] { + f, err := extrafilters.ParseExtraStreamFilters(s) + if err != nil { + return nil, fmt.Errorf("cannot parse extra_stream_filters=%q: %w", s, err) + } + if f != nil { + fs = append(fs, f) + } + } + + return fs, nil +} + func getStringSliceFromRequest(r *http.Request, argName string) ([]string, error) { s := r.FormValue(argName) if s == "" { diff --git a/app/vtselect/traces/tracecommon/tracecommon_test.go b/app/vtselect/traces/tracecommon/tracecommon_test.go new file mode 100644 index 000000000..de3ed371d --- /dev/null +++ b/app/vtselect/traces/tracecommon/tracecommon_test.go @@ -0,0 +1,95 @@ +package tracecommon + +import ( + "context" + "net/http/httptest" + "strings" + "testing" + + "github.com/VictoriaMetrics/VictoriaLogs/lib/logstorage" + + "github.com/VictoriaMetrics/VictoriaTraces/app/vtselect/extrafilters" +) + +func TestGetExtraFilters_Success(t *testing.T) { + f := func(query string, resultExpected []string) { + t.Helper() + + r := httptest.NewRequest("GET", "/select/jaeger/api/services?"+query, nil) + fs, err := getExtraFilters(r) + if err != nil { + t.Fatalf("unexpected error in getExtraFilters: %s", err) + } + result := make([]string, len(fs)) + for i, filter := range fs { + result[i] = filter.String() + } + if len(result) != len(resultExpected) { + t.Fatalf("unexpected number of filters\ngot\n%q\nwant\n%q", result, resultExpected) + } + for i := range result { + if result[i] != resultExpected[i] { + t.Fatalf("unexpected filter\ngot\n%q\nwant\n%q", result, resultExpected) + } + } + } + + // no args + f("", nil) + + // JSON form + f(`extra_filters={"foo":"bar"}`, []string{"foo:=bar"}) + + // LogsQL form + f(`extra_filters=foo:bar`, []string{"foo:bar"}) + + // the arg may repeat, and every filter must be kept + f(`extra_filters={"foo":"bar"}&extra_filters={"baz":"qux"}`, []string{"foo:=bar", "baz:=qux"}) + + // stream filters are parsed too + f(`extra_stream_filters={"foo":"bar"}`, []string{`{foo="bar"}`}) + + // an empty value parses to no filter, so it must not be added + f(`extra_filters=&extra_filters={"foo":"bar"}`, []string{"foo:=bar"}) + + // both args together + f(`extra_filters={"foo":"bar"}&extra_stream_filters={"baz":"qux"}`, []string{"foo:=bar", `{baz="qux"}`}) +} + +func TestGetExtraFilters_Failure(t *testing.T) { + f := func(query string) { + t.Helper() + + r := httptest.NewRequest("GET", "/select/jaeger/api/services?"+query, nil) + if _, err := getExtraFilters(r); err == nil { + t.Fatalf("expecting non-nil error for %q", query) + } + } + + f(`extra_filters={"foo":}`) + f(`extra_filters={"foo":"bar"`) + f(`extra_stream_filters={"foo":}`) +} + +// TestNewQueryContextAppliesExtraFilters checks the funnel which every traces API +// goes through, so a query built by any of them carries the extra filters. +func TestNewQueryContextAppliesExtraFilters(t *testing.T) { + q, err := logstorage.ParseQuery("*") + if err != nil { + t.Fatalf("cannot parse query: %s", err) + } + ef, err := extrafilters.ParseExtraFilters(`{"foo":"bar"}`) + if err != nil { + t.Fatalf("cannot parse extra filters: %s", err) + } + + cp := &CommonParams{ + Query: q, + ExtraFilters: []*logstorage.Filter{ef}, + } + _ = cp.NewQueryContext(context.Background()) + + if !strings.Contains(cp.Query.String(), "foo:=bar") { + t.Fatalf("the extra filter is missing from the query; got %q", cp.Query.String()) + } +} diff --git a/apptest/tests/extra_filters_test.go b/apptest/tests/extra_filters_test.go new file mode 100644 index 000000000..dccf115d0 --- /dev/null +++ b/apptest/tests/extra_filters_test.go @@ -0,0 +1,113 @@ +package tests + +import ( + "os" + "strings" + "testing" + "time" + + at "github.com/VictoriaMetrics/VictoriaTraces/apptest" + otelpb "github.com/VictoriaMetrics/VictoriaTraces/lib/protoparser/opentelemetry/pb" +) + +// TestSingleExtraFiltersJaegerAndTempo checks that the Jaeger and Tempo APIs apply +// the extra_filters query arg. +// +// vmauth injects this arg to keep a user inside its own data, so an API which drops it +// shows that user the data of others. +// See https://github.com/VictoriaMetrics/VictoriaTraces/issues/178 +func TestSingleExtraFiltersJaegerAndTempo(t *testing.T) { + os.RemoveAll(t.Name()) + + tc := at.NewTestCase(t) + defer tc.Stop() + + sut := tc.MustStartVtsingle("vtsingle", []string{ + "-storageDataPath=" + tc.Dir() + "/vtsingle", + "-retentionPeriod=100y", + }) + + spanTime := time.Now() + req := &otelpb.ExportTraceServiceRequest{ + ResourceSpans: []*otelpb.ResourceSpans{ + newExtraFiltersResourceSpans("alpha", "0123456789abcde1", "aa5886e99fffef35a847cb2d493fde10", spanTime), + newExtraFiltersResourceSpans("beta", "0123456789abcde2", "aa5886e99fffef35a847cb2d493fde20", spanTime), + }, + } + sut.OTLPHTTPExportTraces(t, req, at.QueryOpts{}) + sut.ForceFlush(t) + time.Sleep(2 * time.Second) // index will be created after -insert.traceMaxDuration (2s in integration test) + + onlyAlpha := at.QueryOpts{ExtraFilters: []string{`{"resource_attr:service.name":"alpha"}`}} + + // without the filter both services are visible. + tc.Assert(&at.AssertOptions{ + Msg: "unexpected /select/jaeger/api/services response without extra_filters", + Got: func() any { + return sut.JaegerAPIServices(t, at.QueryOpts{}).Data + }, + Want: []string{"alpha", "beta"}, + Retries: 10, + Period: time.Second, + }) + + // with the filter only the allowed service is visible. + tc.Assert(&at.AssertOptions{ + Msg: "extra_filters is ignored by /select/jaeger/api/services", + Got: func() any { + return sut.JaegerAPIServices(t, onlyAlpha).Data + }, + Want: []string{"alpha"}, + Retries: 10, + Period: time.Second, + }) + + // the operations of a service outside the filter must not be returned. + tc.Assert(&at.AssertOptions{ + Msg: "extra_filters is ignored by /select/jaeger/api/services//operations", + Got: func() any { + return len(sut.JaegerAPIOperations(t, "beta", onlyAlpha).Data) + }, + Want: 0, + Retries: 10, + Period: time.Second, + }) + + // the Tempo API shares the same query params, so it must filter too. + tc.Assert(&at.AssertOptions{ + Msg: "extra_filters is ignored by the Tempo tag values API", + Got: func() any { + res := sut.TempoAPITagValues(t, "resource.service.name", onlyAlpha) + return []bool{strings.Contains(res, `"alpha"`), strings.Contains(res, `"beta"`)} + }, + Want: []bool{true, false}, + Retries: 10, + Period: time.Second, + }) +} + +func newExtraFiltersResourceSpans(serviceName, spanID, traceID string, spanTime time.Time) *otelpb.ResourceSpans { + return &otelpb.ResourceSpans{ + Resource: otelpb.Resource{ + Attributes: []*otelpb.KeyValue{ + { + Key: "service.name", + Value: &otelpb.AnyValue{StringValue: &serviceName}, + }, + }, + }, + ScopeSpans: []*otelpb.ScopeSpans{ + { + Spans: []*otelpb.Span{ + { + TraceID: traceID, + SpanID: spanID, + Name: "op-" + serviceName, + StartTimeUnixNano: uint64(spanTime.UnixNano()), + EndTimeUnixNano: uint64(spanTime.Add(time.Second).UnixNano()), + }, + }, + }, + }, + } +} diff --git a/apptest/vtsingle.go b/apptest/vtsingle.go index 8341afd91..e1178ca34 100644 --- a/apptest/vtsingle.go +++ b/apptest/vtsingle.go @@ -32,6 +32,8 @@ type Vtsingle struct { jaegerAPITraceURL string jaegerAPIDependenciesURL string + tempoAPITagValuesURL string + logsQLQueryURL string otlpTracesURL string @@ -73,8 +75,10 @@ func StartVtsingle(instance string, flags []string, cli *Client) (*Vtsingle, err forceFlushURL: fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]), forceMergeURL: fmt.Sprintf("http://%s/internal/force_merge", stderrExtracts[1]), - jaegerAPIServicesURL: fmt.Sprintf("http://%s/select/jaeger/api/services", stderrExtracts[1]), - jaegerAPIOperationsURL: fmt.Sprintf("http://%s/select/jaeger/api/services/%%s/operations", stderrExtracts[1]), + jaegerAPIServicesURL: fmt.Sprintf("http://%s/select/jaeger/api/services", stderrExtracts[1]), + jaegerAPIOperationsURL: fmt.Sprintf("http://%s/select/jaeger/api/services/%%s/operations", stderrExtracts[1]), + + tempoAPITagValuesURL: fmt.Sprintf("http://%s/select/tempo/api/v2/search/tag/%%s/values", stderrExtracts[1]), jaegerAPITracesURL: fmt.Sprintf("http://%s/select/jaeger/api/traces", stderrExtracts[1]), jaegerAPITraceURL: fmt.Sprintf("http://%s/select/jaeger/api/traces/%%s", stderrExtracts[1]), jaegerAPIDependenciesURL: fmt.Sprintf("http://%s/select/jaeger/api/dependencies", stderrExtracts[1]), @@ -117,6 +121,17 @@ func (app *Vtsingle) JaegerAPIServices(t *testing.T, opts QueryOpts) *JaegerAPIS return NewJaegerAPIServicesResponse(t, res) } +// TempoAPITagValues is a test helper function that queries for the values of the given tag +// by sending an HTTP GET request to /select/tempo/api/v2/search/tag//values +// Vtsingle endpoint. It returns the raw response body. +func (app *Vtsingle) TempoAPITagValues(t *testing.T, tagName string, opts QueryOpts) string { + t.Helper() + + url := fmt.Sprintf(app.tempoAPITagValuesURL, tagName) + "?" + opts.asURLValues().Encode() + res, _ := app.cli.Get(t, url) + return res +} + // JaegerAPIOperations is a test helper function that queries for operation list of a service // by sending an HTTP GET request to /select/jaeger/api/services//operations // Vtsingle endpoint. diff --git a/docs/victoriatraces/changelog/CHANGELOG.md b/docs/victoriatraces/changelog/CHANGELOG.md index 0c558cb7d..819ac00b8 100644 --- a/docs/victoriatraces/changelog/CHANGELOG.md +++ b/docs/victoriatraces/changelog/CHANGELOG.md @@ -12,6 +12,8 @@ The following `tip` changes can be tested by building VictoriaTraces components ## tip +* BUGFIX: [Single-node VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) and vtselect in [VictoriaTraces cluster](https://docs.victoriametrics.com/victoriatraces/cluster/): apply the `extra_filters` and `extra_stream_filters` query args to the Jaeger and Tempo APIs. Previously only the native LogsQL endpoints applied them, so a user restricted to its own data via [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) could see the traces of others through the Jaeger and Tempo APIs. See [this issue #178](https://github.com/VictoriaMetrics/VictoriaTraces/issues/178). + * BUGFIX: [cluster version](https://docs.victoriametrics.com/victoriatraces/cluster/): evenly spread rerouted data across available `vtstorage` nodes. Previously, healthy nodes adjacent to unavailable nodes in the `-storageNode` list could receive much more data, resulting in uneven resource usage. See [this issue #228](https://github.com/VictoriaMetrics/VictoriaTraces/issues/228). ## [v0.10.0](https://github.com/VictoriaMetrics/VictoriaTraces/releases/tag/v0.10.0)