diff --git a/cardano-node/ChangeLog.md b/cardano-node/ChangeLog.md index 97413c22f4e..9ea96e31d9d 100644 --- a/cardano-node/ChangeLog.md +++ b/cardano-node/ChangeLog.md @@ -2,6 +2,14 @@ ## Next version +* Added `Cardano.Node.Tracing.Span`: OpenTelemetry-style Begin/End tracing spans + with a shared `trace_id`, `parent_span_id` links for nesting, and a + client-side measured duration shipped in `SpanEnd`. Emits a flat JSON + envelope (`trace_id`, `span_id`, `parent_span_id`, `name`, + `event=begin|end`, `duration_ms`) usable directly from Loki/Tempo, plus a + Prometheus/EKG `spanDurationMs.` metric on end. Nested `withSpan` + calls thread parent context through an explicit `SpanContext`. + ## 11.1.0 -- August 2026 - **Behaviour change:** snapshot options set directly under `LedgerDB` alongside a diff --git a/cardano-node/cardano-node.cabal b/cardano-node/cardano-node.cabal index 82a450478ac..2c2f5b281dc 100644 --- a/cardano-node/cardano-node.cabal +++ b/cardano-node/cardano-node.cabal @@ -106,6 +106,7 @@ library Cardano.Node.Tracing.NodeInfo Cardano.Node.Tracing.NodeStartupInfo Cardano.Node.Tracing.Render + Cardano.Node.Tracing.Span Cardano.Node.Tracing.StateRep Cardano.Node.Tracing.Tracers Cardano.Node.Tracing.Tracers.BlockReplayProgress @@ -240,6 +241,7 @@ test-suite cardano-node-test , cardano-diffusion:{api, cardano-diffusion, orphan-instances} , cardano-node , cardano-slotting + , containers , contra-tracer , directory , filepath @@ -251,7 +253,9 @@ test-suite cardano-node-test , mtl , ouroboros-consensus:{ouroboros-consensus, diffusion} , ouroboros-network:{api, framework, ouroboros-network} + , safe-exceptions , text + , trace-dispatcher , transformers , vector , yaml @@ -262,6 +266,7 @@ test-suite cardano-node-test Test.Cardano.Node.Json Test.Cardano.Node.POM Test.Cardano.Node.TopLevel + Test.Cardano.Node.Tracing.Span Test.Cardano.Tracing.NewTracing.Consistency ghc-options: -threaded -rtsopts "-with-rtsopts=-N -T" diff --git a/cardano-node/src/Cardano/Node/Tracing/Span.hs b/cardano-node/src/Cardano/Node/Tracing/Span.hs new file mode 100644 index 00000000000..2e78b101e60 --- /dev/null +++ b/cardano-node/src/Cardano/Node/Tracing/Span.hs @@ -0,0 +1,240 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | OpenTelemetry-style spans on top of the @trace-dispatcher@ framework. +-- +-- A span is a pair of correlated trace messages — 'SpanBegin' and 'SpanEnd' — +-- sharing a single 'SpanId'. 'withSpan' brackets an action so that the end +-- message is always emitted, even on exception, which is what lets a +-- Loki-style alerting system fire on \"span that never ended\". +-- +-- Spans nest: every span in one logical operation shares the same +-- 'TraceId', and each nested span carries its parent's 'SpanId' as +-- @parent_span_id@. Nesting is threaded through a 'SpanContext'; inner +-- 'withSpan' calls read the currently-active span from the context, attach +-- themselves as its child, and restore the previous state on the way out. +-- +-- The duration is measured /client-side/ (in this process, monotonic clock) +-- and shipped inside 'SpanEnd', so the consumer (cardano-tracer / timeseries / +-- Prometheus) stays stateless — it never has to pair begin\/end itself. +-- +-- * For __timeseries\/Prometheus__: 'asMetrics' emits @spanDurationMs@. +-- * For __Loki (LogQL)__: 'forMachine' emits a flat JSON object with the +-- standard OTel fields @trace_id@, @span_id@, @parent_span_id@ (as +-- lowercase hex strings), @name@, @event=begin|end@, and, on end, +-- @duration_ms@. +-- +-- /Concurrency:/ the 'SpanContext' is a single 'IORef', safe for sequential +-- nesting inside one logical thread of work. If you fork off parallel work +-- that should form its own subtree, snapshot with a fresh 'newSpanContext' +-- (optionally seeded from 'readCurrentSpan') rather than sharing one context +-- across threads. +module Cardano.Node.Tracing.Span + ( SpanId (..) + , TraceId (..) + , SpanTrace (..) + , SpanContext + , newSpanContext + , newSpanId + , newTraceId + , readCurrentSpan + , withSpan + , formatSpanIdHex + , formatTraceIdHex + ) where + +import Cardano.Logging + +import Control.Exception.Safe (MonadMask, finally) +import Control.Monad.IO.Class (MonadIO, liftIO) +import Data.Aeson (Value (String), (.=)) +import Data.IORef (IORef, atomicWriteIORef, newIORef, readIORef) +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Unique (hashUnique, newUnique) +import Data.Word (Word64) +import GHC.Clock (getMonotonicTimeNSec) +import Numeric (showHex) + +-- | 64-bit span identifier. OTel conventionally renders this as +-- 16-character lowercase hex; see 'formatSpanIdHex'. +newtype SpanId = SpanId { unSpanId :: Word64 } + deriving stock (Eq, Ord, Show) + +-- | 128-bit trace identifier shared by every span belonging to the same +-- logical operation. OTel conventionally renders this as 32-character +-- lowercase hex; see 'formatTraceIdHex'. +data TraceId = TraceId !Word64 !Word64 + deriving stock (Eq, Ord, Show) + +-- | The two ends of a span. +-- +-- 'SpanBegin' carries the shared 'TraceId', the span's own 'SpanId', a +-- 'Maybe SpanId' for the parent ('Nothing' at the root of a trace), and +-- the human name of the operation. 'SpanEnd' carries the same 'TraceId' +-- and 'SpanId' plus the measured duration in milliseconds. +data SpanTrace + = SpanBegin !TraceId !SpanId !(Maybe SpanId) !Text + | SpanEnd !TraceId !SpanId !Text !Double + deriving stock (Show) + +-- | Mutable context threaded through nested 'withSpan' calls. Holds the +-- currently-active @(trace_id, span_id)@ so an inner 'withSpan' can attach +-- itself as a child of the outer one without explicit plumbing. +newtype SpanContext = SpanContext + { spanCurrent :: IORef (Maybe (TraceId, SpanId)) + } + +-- | Allocate a fresh top-level context. The first 'withSpan' against it +-- mints a new trace id and starts a root span. +newSpanContext :: MonadIO m => m SpanContext +newSpanContext = liftIO $ SpanContext <$> newIORef Nothing + +-- | Allocate a fresh, process-unique span id. +-- +-- Uses 'Data.Unique' so it needs no extra dependency and never blocks. Ids +-- are unique within a single node run; they are /not/ stable across restarts +-- (fine for correlating one begin with one end — which is all we need). +newSpanId :: MonadIO m => m SpanId +newSpanId = liftIO (SpanId . fromIntegral . hashUnique <$> newUnique) + +-- | Allocate a fresh 128-bit trace id. Uses two consecutive 'Data.Unique' +-- allocations; process-unique, sufficient for downstream correlation. Not +-- cryptographically random. +newTraceId :: MonadIO m => m TraceId +newTraceId = liftIO $ TraceId + <$> (fromIntegral . hashUnique <$> newUnique) + <*> (fromIntegral . hashUnique <$> newUnique) + +-- | Snapshot the span currently active in a context. 'Nothing' when no +-- span is active. Useful in tests and when injecting trace context into +-- a downstream request header (e.g. W3C @traceparent@). +readCurrentSpan :: MonadIO m => SpanContext -> m (Maybe (TraceId, SpanId)) +readCurrentSpan = liftIO . readIORef . spanCurrent + +-- | Run @action@ inside a span, emitting 'SpanBegin' before and 'SpanEnd' +-- after — even if @action@ throws. +-- +-- The span's parent is whatever span is currently active in @ctx@; when +-- nothing is active, a fresh 'TraceId' is minted and this becomes a root +-- span. The context is updated on the way in and restored on the way out, +-- so sequential nesting composes without explicit threading. +-- +-- @ +-- ctx <- newSpanContext +-- withSpan tr ctx \"replayLedger\" $ do +-- ...outer work... +-- withSpan tr ctx \"flushWAL\" $ do +-- ...inner work — its parent_span_id is the outer span's id... +-- @ +withSpan + :: (MonadIO m, MonadMask m) + => Trace m SpanTrace -- ^ where to emit the span messages + -> SpanContext -- ^ nesting context shared by the enclosing scope + -> Text -- ^ human name of the operation + -> m a -- ^ the work to measure + -> m a +withSpan tr ctx name action = do + parent <- readCurrentSpan ctx + (traceId, parentSid) <- case parent of + Nothing -> do !tid <- newTraceId; pure (tid, Nothing) + Just (tid, pid) -> pure (tid, Just pid) + sid <- newSpanId + !t0 <- liftIO getMonotonicTimeNSec + liftIO $ atomicWriteIORef (spanCurrent ctx) (Just (traceId, sid)) + traceWith tr (SpanBegin traceId sid parentSid name) + action `finally` do + !t1 <- liftIO getMonotonicTimeNSec + let !ms = fromIntegral (t1 - t0) / 1_000_000 :: Double + traceWith tr (SpanEnd traceId sid name ms) + liftIO $ atomicWriteIORef (spanCurrent ctx) parent + +-------------------------------------------------------------------------------- +-- OTel-style hex formatting +-------------------------------------------------------------------------------- + +-- | Render a 'SpanId' as a 16-character lowercase hex string, per OTel +-- convention. +formatSpanIdHex :: SpanId -> Text +formatSpanIdHex (SpanId n) = padHex 16 n + +-- | Render a 'TraceId' as a 32-character lowercase hex string, per OTel +-- convention. +formatTraceIdHex :: TraceId -> Text +formatTraceIdHex (TraceId hi lo) = padHex 16 hi <> padHex 16 lo + +padHex :: Int -> Word64 -> Text +padHex width n = + let raw = showHex n "" + pad = replicate (width - length raw) '0' + in Text.pack (pad <> raw) + +-------------------------------------------------------------------------------- +-- Formatting +-------------------------------------------------------------------------------- + +instance LogFormatting SpanTrace where + forMachine _ (SpanBegin tid sid parentSid name) = + mconcat + [ "kind" .= String "SpanBegin" + , "event" .= String "begin" + , "trace_id" .= formatTraceIdHex tid + , "span_id" .= formatSpanIdHex sid + -- 'null' for a root span; a hex string for a nested one. OTel + -- SDKs vary on whether to emit "0000000000000000" or omit the + -- field; 'null' is unambiguous and LogQL handles it cleanly. + , "parent_span_id" .= fmap formatSpanIdHex parentSid + , "name" .= name + ] + forMachine _ (SpanEnd tid sid name ms) = + mconcat + [ "kind" .= String "SpanEnd" + , "event" .= String "end" + , "trace_id" .= formatTraceIdHex tid + , "span_id" .= formatSpanIdHex sid + , "name" .= name + , "duration_ms" .= ms + ] + + forHuman (SpanBegin _ sid _ name) = + "Span begin [" <> formatSpanIdHex sid <> "] " <> name + forHuman (SpanEnd _ sid name ms) = + "Span end [" <> formatSpanIdHex sid <> "] " <> name + <> " (" <> showT ms <> " ms)" + + -- Only the end carries a measurement. The metric name embeds the span name so + -- distinct operations are distinguishable; keep the set of names SMALL to + -- avoid Prometheus/timeseries cardinality blow-up. + asMetrics (SpanBegin{}) = [] + asMetrics (SpanEnd _ _ name ms) = [DoubleM ("spanDurationMs." <> name) ms] + +-------------------------------------------------------------------------------- +-- Documentation / metadata +-------------------------------------------------------------------------------- + +instance MetaTrace SpanTrace where + namespaceFor SpanBegin{} = Namespace [] ["Span", "Begin"] + namespaceFor SpanEnd{} = Namespace [] ["Span", "End"] + + severityFor (Namespace _ ["Span", "Begin"]) _ = Just Info + severityFor (Namespace _ ["Span", "End"]) _ = Just Info + severityFor _ _ = Nothing + + documentFor (Namespace _ ["Span", "Begin"]) = Just + "Start of a correlated span. Carries the shared trace_id, the span's \ + \own span_id, and the parent_span_id (null at the root of a trace)." + documentFor (Namespace _ ["Span", "End"]) = Just + "End of a correlated span. Carries the same trace_id and span_id as \ + \Span.Begin plus the client-side measured duration in milliseconds." + documentFor _ = Nothing + + metricsDocFor (Namespace _ ["Span", "End"]) = + [("spanDurationMs", "Client-side measured span duration, in milliseconds")] + metricsDocFor _ = [] + + allNamespaces = + [ Namespace [] ["Span", "Begin"] + , Namespace [] ["Span", "End"] + ] diff --git a/cardano-node/test/Test/Cardano/Node/Tracing/Span.hs b/cardano-node/test/Test/Cardano/Node/Tracing/Span.hs new file mode 100644 index 00000000000..4fd093b4759 --- /dev/null +++ b/cardano-node/test/Test/Cardano/Node/Tracing/Span.hs @@ -0,0 +1,222 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE TypeApplications #-} + +-- | Hedgehog properties for "Cardano.Node.Tracing.Span". The test surface +-- is kept to the pure API — 'forMachine', 'asMetrics', 'namespaceFor', +-- 'severityFor', hex formatting, plus 'withSpan' round-trip and nesting +-- observed via 'readCurrentSpan'. No capturing tracer is needed: +-- everything nesting-related is checked either by direct value +-- construction (for the JSON shape) or by asking the context which span +-- is active (for the runtime bracket). +module Test.Cardano.Node.Tracing.Span + ( tests + ) where + +import Cardano.Logging (DetailLevel (..), LogFormatting (..), MetaTrace (..), + Metric (..), Namespace (..), SeverityS (..)) +import Cardano.Node.Tracing.Span + +import Control.Exception.Safe (SomeException, try) +import Control.Monad.IO.Class (liftIO) +import qualified Data.Aeson as Aeson +import qualified Data.Aeson.Key as Key +import qualified Data.Aeson.KeyMap as KeyMap +import qualified Data.List as List +import qualified Data.Set as Set +import qualified Data.Text as Text + +import Hedgehog (Property, discover, (===)) +import qualified Hedgehog + + +-- | Two fresh span ids should never collide within one run. +prop_newSpanId_distinct :: Property +prop_newSpanId_distinct = Hedgehog.property $ do + ids <- liftIO $ sequence [newSpanId | _ <- [1 .. (128 :: Int)]] + length (Set.fromList ids) === length ids + +-- | And neither should trace ids — the whole point of a 128-bit trace id +-- is that concurrent operations can't collide. +prop_newTraceId_distinct :: Property +prop_newTraceId_distinct = Hedgehog.property $ do + ids <- liftIO $ sequence [newTraceId | _ <- [1 .. (128 :: Int)]] + length (Set.fromList ids) === length ids + +-- | 'SpanBegin' at the root of a trace carries the six OTel fields +-- Loki/Tempo dashboards read; parent_span_id specifically is 'Null' so +-- consumers can filter for root spans. +prop_forMachine_begin_root_has_expected_fields :: Property +prop_forMachine_begin_root_has_expected_fields = Hedgehog.property $ do + let obj = forMachine DNormal (SpanBegin (TraceId 1 2) (SpanId 3) Nothing "root") + fieldsPresent obj ["kind", "event", "trace_id", "span_id", "parent_span_id", "name"] + KeyMap.lookup (Key.fromString "parent_span_id") obj === Just Aeson.Null + +-- | A nested 'SpanBegin' emits the parent's id as a 16-char hex string. +prop_forMachine_begin_child_has_parent :: Property +prop_forMachine_begin_child_has_parent = Hedgehog.property $ do + let obj = forMachine DNormal + (SpanBegin (TraceId 1 2) (SpanId 3) (Just (SpanId 4)) "child") + KeyMap.lookup (Key.fromString "parent_span_id") obj + === Just (Aeson.String "0000000000000004") + +-- | 'SpanEnd' carries the same identity fields plus the measured +-- duration. +prop_forMachine_end_has_expected_fields :: Property +prop_forMachine_end_has_expected_fields = Hedgehog.property $ do + let obj = forMachine DNormal (SpanEnd (TraceId 1 2) (SpanId 3) "replayLedger" 12.5) + fieldsPresent obj ["kind", "event", "trace_id", "span_id", "name", "duration_ms"] + +-- | Hex widths match OTel conventions exactly: 16 for span, 32 for trace. +prop_span_id_hex_width :: Property +prop_span_id_hex_width = Hedgehog.property $ + Text.length (formatSpanIdHex (SpanId 0xdeadbeef)) === 16 + +prop_trace_id_hex_width :: Property +prop_trace_id_hex_width = Hedgehog.property $ + Text.length (formatTraceIdHex (TraceId 0xaa 0xbb)) === 32 + +-- | 'asMetrics' emits nothing on begin (metrics live only on the end, +-- carrying the measured duration), and emits exactly one gauge on end +-- whose name embeds the span name so distinct operations are visible in +-- Prometheus/EKG without extra state. +prop_asMetrics_begin_empty :: Property +prop_asMetrics_begin_empty = Hedgehog.property $ + asMetrics (SpanBegin (TraceId 0 0) (SpanId 1) Nothing "flushWAL") === ([] :: [Metric]) + +prop_asMetrics_end_emits_duration :: Property +prop_asMetrics_end_emits_duration = Hedgehog.property $ + asMetrics (SpanEnd (TraceId 0 0) (SpanId 1) "flushWAL" 42.0) + === [DoubleM "spanDurationMs.flushWAL" 42.0] + +-- | Namespace / severity metadata drives the trace-dispatcher's routing, +-- so a regression that renames or drops a namespace is a break in the +-- Loki/Prometheus contract, not an internal detail. +prop_namespaceFor_begin :: Property +prop_namespaceFor_begin = Hedgehog.property $ + namespaceFor (SpanBegin (TraceId 0 0) (SpanId 1) Nothing "x") + === (Namespace [] ["Span", "Begin"] :: Namespace SpanTrace) + +prop_namespaceFor_end :: Property +prop_namespaceFor_end = Hedgehog.property $ + namespaceFor (SpanEnd (TraceId 0 0) (SpanId 1) "x" 0) + === (Namespace [] ["Span", "End"] :: Namespace SpanTrace) + +prop_severity_is_info :: Property +prop_severity_is_info = Hedgehog.property $ do + severityFor (Namespace [] ["Span", "Begin"] :: Namespace SpanTrace) Nothing === Just Info + severityFor (Namespace [] ["Span", "End"] :: Namespace SpanTrace) Nothing === Just Info + +prop_allNamespaces_covers_both :: Property +prop_allNamespaces_covers_both = Hedgehog.property $ do + let nss = allNamespaces :: [Namespace SpanTrace] + Hedgehog.assert (Namespace [] ["Span", "Begin"] `elem` nss) + Hedgehog.assert (Namespace [] ["Span", "End"] `elem` nss) + +-- | 'withSpan' returns the action's value on success. The null tracer +-- ('mempty') exercises the bracket without asserting anything about +-- emissions, which are the underlying framework's responsibility. +prop_withSpan_returns_result :: Property +prop_withSpan_returns_result = Hedgehog.property $ do + ctx <- liftIO newSpanContext + r <- liftIO $ withSpan mempty ctx "test" (pure (42 :: Int)) + r === 42 + +-- | 'withSpan' rethrows exceptions from the wrapped action. The +-- 'finally'-based end emission is exercised by GHC's exception +-- semantics; we only need to check that the exception surfaces rather +-- than being swallowed by the bracket. +prop_withSpan_rethrows :: Property +prop_withSpan_rethrows = Hedgehog.property $ do + ctx <- liftIO newSpanContext + result <- liftIO $ try @IO @SomeException $ + withSpan mempty ctx "test" (error "boom" :: IO ()) + case result of + Left _ -> Hedgehog.success + Right _ -> Hedgehog.footnote "exception was swallowed" >> Hedgehog.failure + +-- | A brand-new context has no active span. +prop_context_starts_empty :: Property +prop_context_starts_empty = Hedgehog.property $ do + ctx <- liftIO newSpanContext + cur <- liftIO $ readCurrentSpan ctx + cur === Nothing + +-- | Inside a nested 'withSpan', the inner span sees the same 'TraceId' +-- as the outer one — that is the whole point of trace_id. +prop_nested_shares_trace_id :: Property +prop_nested_shares_trace_id = Hedgehog.property $ do + ctx <- liftIO newSpanContext + (outerTid, innerTid) <- liftIO $ withSpan mempty ctx "outer" $ do + Just (oTid, _) <- readCurrentSpan ctx + iTid <- withSpan mempty ctx "inner" $ do + Just (t, _) <- readCurrentSpan ctx + pure t + pure (oTid, iTid) + outerTid === innerTid + +-- | The inner and outer spans are distinct span ids — nesting doesn't +-- reuse the parent's id. +prop_nested_has_own_span_id :: Property +prop_nested_has_own_span_id = Hedgehog.property $ do + ctx <- liftIO newSpanContext + (outerSid, innerSid) <- liftIO $ withSpan mempty ctx "outer" $ do + Just (_, oSid) <- readCurrentSpan ctx + iSid <- withSpan mempty ctx "inner" $ do + Just (_, s) <- readCurrentSpan ctx + pure s + pure (oSid, iSid) + Hedgehog.assert (outerSid /= innerSid) + +-- | After a nested span ends, the context is restored to the outer +-- span. Without this the second sibling of a nested span would end up +-- attached to the wrong parent. +prop_current_restored_after_inner :: Property +prop_current_restored_after_inner = Hedgehog.property $ do + ctx <- liftIO newSpanContext + (before, after) <- liftIO $ withSpan mempty ctx "outer" $ do + b <- readCurrentSpan ctx + withSpan mempty ctx "inner" (pure ()) + a <- readCurrentSpan ctx + pure (b, a) + before === after + +-- | After the outermost 'withSpan' returns, the context is empty +-- again — root-level bracketing is symmetric. +prop_root_restored_after_outermost :: Property +prop_root_restored_after_outermost = Hedgehog.property $ do + ctx <- liftIO newSpanContext + liftIO $ withSpan mempty ctx "root" (pure ()) + cur <- liftIO $ readCurrentSpan ctx + cur === Nothing + +-- | Two sequential top-level spans on the same context get different +-- trace ids — each is the root of its own trace. +prop_sequential_roots_have_different_trace_ids :: Property +prop_sequential_roots_have_different_trace_ids = Hedgehog.property $ do + ctx <- liftIO newSpanContext + t1 <- liftIO $ withSpan mempty ctx "first" $ do + Just (t, _) <- readCurrentSpan ctx + pure t + t2 <- liftIO $ withSpan mempty ctx "second" $ do + Just (t, _) <- readCurrentSpan ctx + pure t + Hedgehog.assert (t1 /= t2) + + +-- | Helper: assert that a 'forMachine' object contains every named key. +fieldsPresent :: KeyMap.KeyMap v -> [String] -> Hedgehog.PropertyT IO () +fieldsPresent obj keys = do + let actual = List.sort (map show (KeyMap.keys obj)) + missing = filter (\k -> not (KeyMap.member (Key.fromString k) obj)) keys + case missing of + [] -> Hedgehog.success + _ -> do + Hedgehog.footnote ("missing fields: " <> show missing <> ", actual keys: " <> show actual) + Hedgehog.failure + + +tests :: IO Bool +tests = + Hedgehog.checkParallel $$discover diff --git a/cardano-node/test/cardano-node-test.hs b/cardano-node/test/cardano-node-test.hs index 0706e005edb..43d6462ae4a 100644 --- a/cardano-node/test/cardano-node-test.hs +++ b/cardano-node/test/cardano-node-test.hs @@ -14,6 +14,7 @@ import qualified Test.Cardano.Node.FilePermissions import qualified Test.Cardano.Node.Json import qualified Test.Cardano.Node.POM import qualified Test.Cardano.Node.TopLevel +import qualified Test.Cardano.Node.Tracing.Span import qualified Test.Cardano.Tracing.NewTracing.Consistency import qualified Cardano.Crypto.Init as Crypto @@ -35,5 +36,6 @@ main = do , Test.Cardano.Node.Json.tests , Test.Cardano.Node.POM.tests , Test.Cardano.Node.TopLevel.tests + , Test.Cardano.Node.Tracing.Span.tests , Test.Cardano.Tracing.NewTracing.Consistency.tests ]