diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 18801ac82..45c94ddca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,6 +118,16 @@ jobs: env: USE_TRANSPARENT_COMPILER: ${{ matrix.use-transparent-compiler }} USE_WORKSPACE_LOADER: ${{ matrix.workspace-loader }} + FAILED_TRACES_DIR: ${{ github.workspace }}/failed_traces + + - name: Upload failed test traces + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-test-traces-${{ matrix.os }}-${{ matrix.label }}-${{ matrix.workspace-loader }}-${{ matrix.use-transparent-compiler }} + path: ${{ github.workspace }}/failed_traces/ + if-no-files-found: ignore + retention-days: 7 analyze: runs-on: ubuntu-latest diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..767b38673 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,47 @@ +services: + seq: + profiles: + - seq + image: datalust/seq:2024.3 + ports: + - 5341:80 # http and collection + environment: + - ACCEPT_EULA=Y + # http://localhost:5341 + # OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:5341/ingest/otlp/v1/traces + # OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf" + # OTEL_EXPORTER_OTLP_HEADERS="X-Seq-ApiKey=your_api_key" + jaeger: + profiles: + - jaeger + image: jaegertracing/all-in-one + ports: + - 6831:6831/udp + - 6832:6832/udp + - 5778:5778 + - 16686:16686 + - 4317:4317 + - 4318:4318 + - 14250:14250 + - 14268:14268 + - 14269:14269 + - 9411:9411 + environment: + - COLLECTOR_ZIPKIN_HTTP_PORT=9411 + - COLLECTOR_OTLP_ENABLED=true + # http://localhost:16686/ + # OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 + aspire: + profiles: + - aspire + image: mcr.microsoft.com/dotnet/nightly/aspire-dashboard + ports: + - 4317:18889 #otel + - 4318:18890 + - 18888:18888 #http + environment: + - DOTNET_DASHBOARD_UNSECURED_ALLOW_ANONYMOUS=true + # - OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 + # - OTEL_EXPORTER_OTLP_HEADERS=X-Seq-ApiKey=your_api_key + # http://localhost:18888/ + # OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 diff --git a/docs/Analyzing failed test traces.md b/docs/Analyzing failed test traces.md new file mode 100644 index 000000000..c15eccdb6 --- /dev/null +++ b/docs/Analyzing failed test traces.md @@ -0,0 +1,260 @@ +# Analyzing Failed Test Traces + +When tests fail in CI, FsAutoComplete exports OpenTelemetry traces for the failed tests in OTLP JSON format. These traces can be imported into observability tools like Jaeger, Aspire Dashboard, or Zipkin for detailed analysis. + +## Finding the Trace Files + +### GitHub Actions + +1. Navigate to the failed workflow run +2. Scroll to the **Artifacts** section at the bottom +3. Download the artifact named `failed-test-traces-{os}-{dotnet}-{loader}-{compiler}` +4. Extract the ZIP file to find `failed_tests_{timestamp}_{process-id}.otlp.json`. + +### Local Development + +When running tests locally with `CI=true`: + +```bash +CI=true FAILED_TRACES_DIR=./traces dotnet run -c Release -f net8.0 --project test/FsAutoComplete.Tests.Lsp/FsAutoComplete.Tests.Lsp.fsproj +``` + +Trace files will be written to the specified directory (default: `failed_traces/`). + +## Viewing Traces + +### Option 1: Jaeger + +[Jaeger](https://www.jaegertracing.io/) is a popular open-source distributed tracing platform. + +#### Quick Start with Docker + +```bash +# Start Jaeger with OTLP support +docker run -d --name jaeger \ + -p 16686:16686 \ + -p 4317:4317 \ + -p 4318:4318 \ + jaegertracing/all-in-one:latest +``` + +#### Import Traces via OTLP HTTP + +```bash +# Send the trace file to Jaeger's OTLP HTTP endpoint +curl -X POST http://localhost:4318/v1/traces \ + -H "Content-Type: application/json" \ + -d @failed_tests_20251218_100000_123_1234.otlp.json +``` + +#### View in Jaeger UI + +1. Open http://localhost:16686 +2. Select service `FsAutoComplete.Tests.Lsp` +3. Click "Find Traces" +4. Click on a trace to see the span details, including: + - Test name + - Duration + - Error status and message + - Exception details (type, message, stack trace) + - Source code location (file path, line number) + +### Option 2: Aspire Dashboard + +The [.NET Aspire Dashboard](https://learn.microsoft.com/en-us/dotnet/aspire/fundamentals/dashboard/overview) provides a standalone trace viewer. + +#### Quick Start with Docker + +```bash +# Start Aspire Dashboard +docker run -d --name aspire-dashboard \ + -p 18888:18888 \ + -p 4317:18889 \ + -p 4318:18890 \ + mcr.microsoft.com/dotnet/aspire-dashboard:latest +``` + +#### Import Traces + +```bash +# Send traces to Aspire's OTLP HTTP endpoint +curl -X POST http://localhost:4318/v1/traces \ + -H "Content-Type: application/json" \ + -d @failed_tests_20251218_100000_123_1234.otlp.json +``` + +#### View in Aspire Dashboard + +1. Open http://localhost:18888 +2. Navigate to the "Traces" tab +3. Browse and filter traces by service name or status + +### Option 3: OpenTelemetry Collector + +For more flexibility, use the [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) to route traces to multiple backends. + +#### Collector Configuration + +Create `otel-collector-config.yaml`: + +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +exporters: + # Export to Jaeger + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + + # Or export to console for debugging + debug: + verbosity: detailed + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [otlp/jaeger, debug] +``` + +#### Run the Collector + +```bash +docker run -d --name otel-collector \ + -p 4317:4317 \ + -p 4318:4318 \ + -v $(pwd)/otel-collector-config.yaml:/etc/otelcol/config.yaml \ + otel/opentelemetry-collector:latest +``` + +### Option 4: Programmatic Analysis + +You can also analyze the JSON file programmatically: + +#### F# Example + +```fsharp +open System.Text.Json + +type AnyValue = { stringValue: string option; intValue: string option } +type KeyValue = { key: string; value: AnyValue } +type SpanStatus = { code: int; message: string option } +type OtlpSpan = { + traceId: string + spanId: string + name: string + startTimeUnixNano: string + endTimeUnixNano: string + attributes: KeyValue[] + status: SpanStatus +} +type ScopeSpans = { spans: OtlpSpan[] } +type ResourceSpans = { scopeSpans: ScopeSpans[] } +type TracesData = { resourceSpans: ResourceSpans[] } + +let traces = JsonSerializer.Deserialize(File.ReadAllText("failed_tests.otlp.json")) + +for rs in traces.resourceSpans do + for ss in rs.scopeSpans do + for span in ss.spans do + printfn "Test: %s" span.name + printfn "Status: %s" (if span.status.code = 2 then "FAILED" else "OK") + for attr in span.attributes do + match attr.value.stringValue with + | Some v -> printfn " %s: %s" attr.key v + | None -> () +``` + +#### PowerShell Example + +```powershell +$traces = Get-Content "failed_tests.otlp.json" | ConvertFrom-Json + +foreach ($rs in $traces.resourceSpans) { + foreach ($ss in $rs.scopeSpans) { + foreach ($span in $ss.spans) { + Write-Host "Test: $($span.name)" + Write-Host "Status: $(if ($span.status.code -eq 2) { 'FAILED' } else { 'OK' })" + foreach ($attr in $span.attributes) { + if ($attr.value.stringValue) { + Write-Host " $($attr.key): $($attr.value.stringValue)" + } + } + Write-Host "" + } + } +} +``` + +## Trace Structure + +The trace files follow the [OTLP JSON Protobuf Encoding](https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding) specification. + +Each file contains the failed test span and its child spans. The failed test span contains these fields: + +| Field | Description | +|-------|-------------| +| `traceId` | Unique trace identifier (32-char lowercase hex string) | +| `spanId` | Unique span identifier (16-char lowercase hex string) | +| `name` | Full test name (e.g., `FSAC.lsp.CodeFix.TestName`) | +| `startTimeUnixNano` | Test start time in nanoseconds since Unix epoch | +| `endTimeUnixNano` | Test end time in nanoseconds since Unix epoch | +| `status.code` | `0` = Unset, `1` = OK, `2` = Error | +| `status.message` | Error description | + +### Common Attributes + +| Attribute | Description | +|-----------|-------------| +| `test.result.status` | `Passed`, `Failed`, `Error`, or `Ignored` | +| `test.result.message` | Detailed failure message | +| `code.filepath` | Source file path | +| `code.lineno` | Line number in source | + +### Events + +Exception details are recorded as span events with the name `exception`, containing: +- `exception.type` +- `exception.message` +- `exception.stacktrace` + +## Troubleshooting + +### No trace file generated + +- Ensure `CI=true` environment variable is set +- Check that tests actually failed (traces are only generated for failures) +- Verify `FAILED_TRACES_DIR` points to a writable directory + +### Traces not appearing in Jaeger/Aspire + +- Verify the OTLP endpoint is accessible +- Check that the JSON file is valid: `jq . failed_tests.otlp.json` +- Ensure the Content-Type header is set to `application/json` + +### Large trace files + +If many tests fail, the trace file may be large. Consider: +- Filtering to specific test categories +- Using `--filter` to run a subset of tests +- Compressing before upload: `gzip failed_tests.otlp.json` + +## Implementation Notes + +The OTLP JSON serialization is implemented in `test/FsAutoComplete.Tests.Lsp/OpenTelemetry.Exporter.fs`. + +While the `OpenTelemetry.Exporter.OpenTelemetryProtocol` NuGet package contains protobuf types (`OpenTelemetry.Proto.Trace.V1.Span`, etc.), these types are marked as **internal** and cannot be used directly by consuming code. Therefore, we implement our own JSON serialization following the OTLP specification. + +Key implementation details: +- Trace and span IDs use **lowercase hex encoding** (not base64) per OTLP JSON spec +- Timestamps are nanoseconds since Unix epoch as strings +- Enum values are serialized as integers (e.g., `status.code: 2` for Error) +- The exporter keeps a trace when its test span has a `Failed` or `Error` status. +- The exporter removes traces for passed and ignored tests. diff --git a/src/FsAutoComplete.Core/CompilerServiceInterface.fs b/src/FsAutoComplete.Core/CompilerServiceInterface.fs index b2f0b2cd9..e9b5342ff 100644 --- a/src/FsAutoComplete.Core/CompilerServiceInterface.fs +++ b/src/FsAutoComplete.Core/CompilerServiceInterface.fs @@ -1,5 +1,7 @@ namespace FsAutoComplete +open FsAutoComplete.Utils.Tracing +open FsAutoComplete.Telemetry open System.IO open FSharp.Compiler.CodeAnalysis open Utils @@ -14,6 +16,7 @@ open System open FsToolkit.ErrorHandling open FSharp.Compiler.CodeAnalysis.ProjectSnapshot open System.Threading +open IcedTasks type Version = int @@ -107,6 +110,8 @@ type FSharpCompilerServiceChecker ?transparentCompilerCacheSizes = cacheSize ) + let thisType = typeof + let entityCache = EntityCache() // FCS can't seem to handle parallel project restores for script files @@ -333,15 +338,20 @@ type FSharpCompilerServiceChecker } member self.GetProjectSnapshotsFromScript(file: string, source, tfm: FSIRefs.TFM) = - async { - try - do! scriptLocker.WaitAsync() |> Async.AwaitTask + asyncEx { + let tags = + seq { + yield "file", box file + yield "tfm", tfm + } + + use _trace = Tracing.fsacActivitySource.StartActivityForType(thisType, tags = tags) + use! _l = scriptLocker.LockAsync() + + match tfm with + | FSIRefs.TFM.NetFx -> return! self.GetNetFxScriptSnapshot(file, source) + | FSIRefs.TFM.NetCore -> return! self.GetNetCoreScriptSnapshot(file, source) - match tfm with - | FSIRefs.TFM.NetFx -> return! self.GetNetFxScriptSnapshot(file, source) - | FSIRefs.TFM.NetCore -> return! self.GetNetCoreScriptSnapshot(file, source) - finally - scriptLocker.Release() |> ignore } @@ -414,15 +424,19 @@ type FSharpCompilerServiceChecker } member self.GetProjectOptionsFromScript(file: string, source, tfm) = - async { - try - do! scriptLocker.WaitAsync() |> Async.AwaitTask - - match tfm with - | FSIRefs.TFM.NetFx -> return! self.GetNetFxScriptOptions(file, source) - | FSIRefs.TFM.NetCore -> return! self.GetNetCoreScriptOptions(file, source) - finally - scriptLocker.Release() |> ignore + asyncEx { + let tags = + seq { + yield "file", box file + yield "tfm", box tfm + } + + use _trace = Tracing.fsacActivitySource.StartActivityForType(thisType, tags = tags) + use! _l = scriptLocker.LockAsync() + + match tfm with + | FSIRefs.TFM.NetFx -> return! self.GetNetFxScriptOptions(file, source) + | FSIRefs.TFM.NetCore -> return! self.GetNetCoreScriptOptions(file, source) } diff --git a/src/FsAutoComplete.Logging/FsOpenTelemetry.fs b/src/FsAutoComplete.Logging/FsOpenTelemetry.fs index 108cb912f..1e813db1f 100644 --- a/src/FsAutoComplete.Logging/FsOpenTelemetry.fs +++ b/src/FsAutoComplete.Logging/FsOpenTelemetry.fs @@ -574,13 +574,15 @@ type ActivityExtensions = let tags = ActivityTagsCollection( - [ yield KeyValuePair(SemanticConventions.General.Exceptions.exception_escaped, box escaped) + seq { + yield KeyValuePair(SemanticConventions.General.Exceptions.exception_escaped, box escaped) yield KeyValuePair(SemanticConventions.General.Exceptions.exception_type, box errorType) if Option.isSome stacktrace then yield KeyValuePair(SemanticConventions.General.Exceptions.exception_stacktrace, box stacktrace.Value) - yield KeyValuePair(SemanticConventions.General.Exceptions.exception_message, box errorMessage) ] + yield KeyValuePair(SemanticConventions.General.Exceptions.exception_message, box errorMessage) + } ) ActivityEvent(SemanticConventions.General.Exceptions.exception_, tags = tags) @@ -602,11 +604,14 @@ type ActivityExtensions = let tags = ActivityTagsCollection( - [ yield KeyValuePair(SemanticConventions.General.Exceptions.exception_escaped, box escaped) + seq { + yield KeyValuePair(SemanticConventions.General.Exceptions.exception_escaped, box escaped) yield KeyValuePair(SemanticConventions.General.Exceptions.exception_type, box exceptionType) yield KeyValuePair(SemanticConventions.General.Exceptions.exception_stacktrace, box exceptionStackTrace) + if not <| String.IsNullOrEmpty(exceptionMessage) then - yield KeyValuePair(SemanticConventions.General.Exceptions.exception_message, box exceptionMessage) ] + yield KeyValuePair(SemanticConventions.General.Exceptions.exception_message, box exceptionMessage) + } ) ActivityEvent(SemanticConventions.General.Exceptions.exception_, tags = tags) diff --git a/src/FsAutoComplete/LspServers/AdaptiveFSharpLspServer.fs b/src/FsAutoComplete/LspServers/AdaptiveFSharpLspServer.fs index cd6f13137..135289119 100644 --- a/src/FsAutoComplete/LspServers/AdaptiveFSharpLspServer.fs +++ b/src/FsAutoComplete/LspServers/AdaptiveFSharpLspServer.fs @@ -673,6 +673,14 @@ type AdaptiveFSharpLspServer { x with Change = Some TextDocumentSyncKind.Incremental } | x -> x) + DiagnosticProvider = + Some( + U2.C1 + { WorkDoneProgress = Some false + Identifier = None + InterFileDependencies = true + WorkspaceDiagnostics = false } + ) InlineValueProvider = inlineValueToggle |> Option.map U3.C2 } let response: Ionide.LanguageServerProtocol.Types.InitializeResult = @@ -2695,7 +2703,38 @@ type AdaptiveFSharpLspServer override x.TextDocumentDeclaration p = x.logUnimplementedRequest p - override x.TextDocumentDiagnostic p = x.logUnimplementedRequest p + override x.TextDocumentDiagnostic p = + asyncResult { + let tags = [ "DocumentDiagnosticParams", box p ] + use trace = fsacActivitySource.StartActivityForType(thisType, tags = tags) + + try + logger.info ( + Log.setMessage "TextDocumentDiagnostic Request: {params}" + >> Log.addContextDestructured "params" p + ) + + let filePath = p.TextDocument.GetFilePath() |> Utils.normalizePath + + let! diags = state.GetDiagnostics filePath |> AsyncResult.ofStringErr + + return + DocumentDiagnosticReport.C1( + { Kind = "full" + ResultId = None + Items = diags + RelatedDocuments = None } + ) + + with e -> + trace |> Tracing.recordException e + + let logCfg = + Log.setMessage "TextDocumentDiagnostic Request Errored {p}" + >> Log.addContextDestructured "p" p + + return! returnException e logCfg + } override x.TextDocumentLinkedEditingRange p = x.logUnimplementedRequest p diff --git a/src/FsAutoComplete/LspServers/AdaptiveServerState.fs b/src/FsAutoComplete/LspServers/AdaptiveServerState.fs index b3b087e75..9a11f6537 100644 --- a/src/FsAutoComplete/LspServers/AdaptiveServerState.fs +++ b/src/FsAutoComplete/LspServers/AdaptiveServerState.fs @@ -471,7 +471,8 @@ type AdaptiveState let filePathUntag = UMX.untag filePath let source = file.Source let fileName = Path.GetFileName filePathUntag - + let tags = seq { "filePath", box filePath } + use _t = fsacActivitySource.StartActivityForType(thisType, tags = tags) let inline getSourceLine lineNo = (source: ISourceText).GetLineString(lineNo - 1) @@ -489,12 +490,12 @@ type AdaptiveState UnusedOpens.getUnusedOpens (tyRes.GetCheckResults, getSourceLine) |> Async.withCancellation progress.CancellationToken - do! - triggerNotificationAndWait - (NotificationEvent.UnusedOpens(filePath, (unused |> List.toArray), file.Version)) - ct + return + NotificationEvent.UnusedOpens(filePath, (unused |> List.toArray), file.Version) + |> Some with e -> logger.error (Log.setMessage "checkUnusedOpens failed" >> Log.addExn e) + return None } let checkUnusedDeclarations = @@ -515,9 +516,10 @@ type AdaptiveState let unused = unused |> Seq.toArray - do! triggerNotificationAndWait (NotificationEvent.UnusedDeclarations(filePath, unused, file.Version)) ct + return NotificationEvent.UnusedDeclarations(filePath, unused, file.Version) |> Some with e -> logger.error (Log.setMessage "checkUnusedDeclarations failed" >> Log.addExn e) + return None } let checkSimplifiedNames = @@ -535,9 +537,10 @@ type AdaptiveState |> Async.withCancellation progress.CancellationToken let simplified = Array.ofSeq simplified - do! triggerNotificationAndWait (NotificationEvent.SimplifyNames(filePath, simplified, file.Version)) ct + return NotificationEvent.SimplifyNames(filePath, simplified, file.Version) |> Some with e -> logger.error (Log.setMessage "checkSimplifiedNames failed" >> Log.addExn e) + return None } let checkUnnecessaryParentheses = @@ -568,12 +571,12 @@ type AdaptiveState | _ -> ranges) - do! - triggerNotificationAndWait - (NotificationEvent.UnnecessaryParentheses(filePath, Array.ofSeq unnecessaryParentheses, file.Version)) - ct + return + NotificationEvent.UnnecessaryParentheses(filePath, Array.ofSeq unnecessaryParentheses, file.Version) + |> Some with e -> logger.error (Log.setMessage "checkUnnecessaryParentheses failed" >> Log.addExn e) + return None } let inline isNotExcluded (exclusions: Regex array) = @@ -601,7 +604,10 @@ type AdaptiveState then checkUnnecessaryParentheses ] - async { do! analyzers |> Async.parallel75 |> Async.Ignore } + async { + let! results = analyzers |> Async.parallel75 + return results |> Array.choose id + } let tryUriCreate (s: string) = match Uri.TryCreate(s, UriKind.Absolute) with @@ -617,10 +623,14 @@ type AdaptiveState (compilerOptions: CompilerProjectOption) = asyncEx { + use _t = + fsacActivitySource.StartActivityForType(thisType, tags = seq { "filePath", box volatileFile.FileName }) + if config.EnableAnalyzers then let file = volatileFile.FileName try + use! _l = analyzersLocker.LockAsync() use progress = new ServerProgressReport(lspClient) if config.Notifications.BackgroundServiceProgress then @@ -675,20 +685,93 @@ type AdaptiveState analyzerPredicate ) - let! ct = Async.CancellationToken - do! triggerNotificationAndWait (NotificationEvent.AnalyzerMessage(res, file, volatileFile.Version)) ct - Loggers.analyzers.info (Log.setMessageI $"end analysis of {file:file}") + return NotificationEvent.AnalyzerMessage(res, file, volatileFile.Version) |> Some | _ -> Loggers.analyzers.info (Log.setMessageI $"missing components of {file:file} to run analyzers, skipped them") - - () + return None with ex -> Loggers.analyzers.error (Log.setMessageI $"Run failed for {file:file}" >> Log.addExn ex) + return None + else + return None } - + let unusedOpensToDiagnostic n = + { Range = fcsRangeToLsp n + Code = Some(U2.C2 "FSAC0001") + Severity = Some DiagnosticSeverity.Hint + Source = Some "FSAC" + Message = "Unused open statement" + RelatedInformation = None + Tags = Some [| DiagnosticTag.Unnecessary |] + Data = None + CodeDescription = None } + + let unusedDeclarationsToDiagnostic n = + { Range = fcsRangeToLsp n + Code = Some(U2.C2 "FSAC0003") + Severity = Some DiagnosticSeverity.Hint + Source = Some "FSAC" + Message = "This value is unused" + RelatedInformation = Some [||] + Tags = Some [| DiagnosticTag.Unnecessary |] + Data = None + CodeDescription = None } + + let simplifyNamesToDiagnostic (r: FSharp.Compiler.EditorServices.SimplifyNames.SimplifiableRange) = + { Diagnostic.Range = fcsRangeToLsp r.Range + Code = Some(U2.C2 "FSAC0002") + Severity = Some DiagnosticSeverity.Hint + Source = Some "FSAC" + Message = "This qualifier is redundant" + RelatedInformation = Some [||] + Tags = Some [| DiagnosticTag.Unnecessary |] + Data = None + CodeDescription = None } + + let unnecessaryParenthesesToDiagnostic r = + { Diagnostic.Range = fcsRangeToLsp r + Code = Some(U2.C2 "FSAC0004") + Severity = Some DiagnosticSeverity.Hint + Source = Some "FSAC" + Message = "Parentheses can be removed" + RelatedInformation = Some [||] + Tags = Some [| DiagnosticTag.Unnecessary |] + Data = None + CodeDescription = None } + + let analyzersToDiagnostic (m: FSharp.Analyzers.SDK.Message) = + let range = fcsRangeToLsp m.Range + + let severity = + match m.Severity with + | FSharp.Analyzers.SDK.Severity.Hint -> DiagnosticSeverity.Hint + | FSharp.Analyzers.SDK.Severity.Info -> DiagnosticSeverity.Information + | FSharp.Analyzers.SDK.Severity.Warning -> DiagnosticSeverity.Warning + | FSharp.Analyzers.SDK.Severity.Error -> DiagnosticSeverity.Error + + let fixes = + match m.Fixes with + | [] -> None + | fixes -> + fixes + |> List.map (fun fix -> + { Range = fcsRangeToLsp fix.FromRange + NewText = fix.ToText }) + |> Ionide.LanguageServerProtocol.Server.serialize + |> Some + + { Range = range + Code = Option.ofObj m.Code |> Option.map U2.C2 + Severity = Some severity + Source = Some $"F# Analyzers (%s{m.Type})" + Message = m.Message + RelatedInformation = None + Tags = None + CodeDescription = None + Data = fixes } let handleCommandEvents (n: NotificationEvent, ct: CancellationToken, completion: TaskCompletionSource option) = try @@ -724,76 +807,28 @@ type AdaptiveState | NotificationEvent.UnusedOpens(file, opens, version) -> let uri = Path.LocalPathToUri file - let diags = - opens - |> Array.map (fun n -> - { Range = fcsRangeToLsp n - Code = Some(U2.C2 "FSAC0001") - Severity = Some DiagnosticSeverity.Hint - Source = Some "FSAC" - Message = "Unused open statement" - RelatedInformation = None - Tags = Some [| DiagnosticTag.Unnecessary |] - Data = None - CodeDescription = None }) + let diags = opens |> Array.map unusedOpensToDiagnostic do! diagnosticCollections.SetForAndWait(uri, "F# Unused opens", version, diags) | NotificationEvent.UnusedDeclarations(file, decls, version) -> let uri = Path.LocalPathToUri file - let diags = - decls - |> Array.map (fun n -> - { Range = fcsRangeToLsp n - Code = Some(U2.C2 "FSAC0003") - Severity = Some DiagnosticSeverity.Hint - Source = Some "FSAC" - Message = "This value is unused" - RelatedInformation = Some [||] - Tags = Some [| DiagnosticTag.Unnecessary |] - Data = None - CodeDescription = None }) + let diags = decls |> Array.map unusedDeclarationsToDiagnostic do! diagnosticCollections.SetForAndWait(uri, "F# Unused declarations", version, diags) | NotificationEvent.SimplifyNames(file, decls, version) -> let uri = Path.LocalPathToUri file - let diags = - decls - |> Array.map - - (fun - ({ Range = range - RelativeName = _relName }) -> - { Diagnostic.Range = fcsRangeToLsp range - Code = Some(U2.C2 "FSAC0002") - Severity = Some DiagnosticSeverity.Hint - Source = Some "FSAC" - Message = "This qualifier is redundant" - RelatedInformation = Some [||] - Tags = Some [| DiagnosticTag.Unnecessary |] - Data = None - CodeDescription = None }) + let diags = decls |> Array.map simplifyNamesToDiagnostic do! diagnosticCollections.SetForAndWait(uri, "F# simplify names", version, diags) | NotificationEvent.UnnecessaryParentheses(file, ranges, version) -> let uri = Path.LocalPathToUri file - let diags = - ranges - |> Array.map (fun range -> - { Diagnostic.Range = fcsRangeToLsp range - Code = Some(U2.C2 "FSAC0004") - Severity = Some DiagnosticSeverity.Hint - Source = Some "FSAC" - Message = "Parentheses can be removed" - RelatedInformation = Some [||] - Tags = Some [| DiagnosticTag.Unnecessary |] - Data = None - CodeDescription = None }) + let diags = ranges |> Array.map unnecessaryParenthesesToDiagnostic do! diagnosticCollections.SetForAndWait(uri, "F# unnecessary parentheses", version, diags) @@ -846,38 +881,7 @@ type AdaptiveState match messages with | [||] -> do! diagnosticCollections.SetForAndWait(uri, "F# Analyzers", version, [||]) | messages -> - let diags = - messages - |> Array.map (fun m -> - let range = fcsRangeToLsp m.Range - - let severity = - match m.Severity with - | FSharp.Analyzers.SDK.Severity.Hint -> DiagnosticSeverity.Hint - | FSharp.Analyzers.SDK.Severity.Info -> DiagnosticSeverity.Information - | FSharp.Analyzers.SDK.Severity.Warning -> DiagnosticSeverity.Warning - | FSharp.Analyzers.SDK.Severity.Error -> DiagnosticSeverity.Error - - let fixes = - match m.Fixes with - | [] -> None - | fixes -> - fixes - |> List.map (fun fix -> - { Range = fcsRangeToLsp fix.FromRange - NewText = fix.ToText }) - |> Ionide.LanguageServerProtocol.Server.serialize - |> Some - - { Range = range - Code = Option.ofObj m.Code |> Option.map U2.C2 - Severity = Some severity - Source = Some $"F# Analyzers (%s{m.Type})" - Message = m.Message - RelatedInformation = None - Tags = None - CodeDescription = None - Data = fixes }) + let diags = messages |> Array.map analyzersToDiagnostic do! diagnosticCollections.SetForAndWait(uri, "F# Analyzers", version, diags) | NotificationEvent.TestDetected(file, tests) -> @@ -1306,11 +1310,15 @@ type AdaptiveState <| fileChecked.Publish.Subscribe(fun (checkedFile) -> async { if checkedFile.VolatileFile.Source.Length > 0 then + let! ct = Async.CancellationToken let config = config |> AVal.force let analyzerPaths = analyzerPaths |> AVal.force - do! builtInCompilerAnalyzers config checkedFile.VolatileFile checkedFile.ParseAndCheckResults + let! results = builtInCompilerAnalyzers config checkedFile.VolatileFile checkedFile.ParseAndCheckResults - do! + for result in results do + do! triggerNotificationAndWait result ct + + match! runAnalyzers config analyzerPaths @@ -1318,6 +1326,9 @@ type AdaptiveState checkedFile.VolatileFile checkedFile.Options checkedFile.CompilerOptions + with + | None -> () + | Some result -> do! triggerNotificationAndWait result ct do! lspClient.NotifyDocumentAnalyzed @@ -1468,11 +1479,7 @@ type AdaptiveState | CompilerProjectOption.TransparentCompiler snap -> taskResult { return! checker.ParseFile(file.FileName, snap) } | CompilerProjectOption.BackgroundCompiler opts -> - taskResult { - - - return! checker.ParseFile(file.FileName, file.Source, opts) - } + taskResult { return! checker.ParseFile(file.FileName, file.Source, opts) } let! ct = Async.CancellationToken @@ -1524,6 +1531,14 @@ type AdaptiveState let! projs = asyncResult { + let tags = + seq { + yield "filePath", box filePath + yield "version", file.Version + yield "lastTouched", file.LastTouched + } + + use _trace = fsacActivitySource.StartActivityForType(thisType, tags = tags) let cts = getOpenFileTokenOrDefault filePath use linkedCts = CancellationTokenSource.CreateLinkedTokenSource(ctok, cts) @@ -1688,7 +1703,6 @@ type AdaptiveState |> Async.parallel75 } - let getAllFilesToProjectOptionsSelected () = async { let! set = getAllFilesToProjectOptions () @@ -1975,7 +1989,12 @@ type AdaptiveState } let forceGetOpenFileTypeCheckResults (filePath: string) = - getOpenFileTypeCheckResults (filePath) |> AsyncAVal.forceAsync + async { + use _t = + fsacActivitySource.StartActivityForType(thisType, tags = seq { "filePath", box filePath }) + + return! getOpenFileTypeCheckResults (filePath) |> AsyncAVal.forceAsync + } @@ -2593,7 +2612,6 @@ type AdaptiveState } - member x.RootPath with get () = AVal.force rootPath and set v = transact (fun () -> rootPath.Value <- v) @@ -2885,6 +2903,45 @@ type AdaptiveState member x.CancelServerProgress(progressToken: ProgressToken) = progressLookup.Cancel progressToken + member x.GetDiagnostics(file: string) = + asyncResult { + let! check = forceGetOpenFileTypeCheckResults file + let! proj = forceGetProjectOptions file + let! file = x.GetOpenFileOrRead file + let config = x.Config + let analyzerPaths = analyzerPaths |> AVal.force + let! buildInAnalyzer = builtInCompilerAnalyzers config file check + + let! externalAnalyzer = + runAnalyzers config analyzerPaths check file proj (AVal.force proj.FSharpProjectCompilerOptions) + + let fcsDiags = + Array.append check.GetParseResults.Diagnostics check.GetCheckResults.Diagnostics + |> Array.distinctBy (fun error -> + error.Severity, + error.ErrorNumber, + error.StartLine, + error.StartColumn, + error.EndLine, + error.EndColumn, + error.Message) + |> Array.map fcsErrorToDiagnostic + + let analyzerDiags = + [| yield! buildInAnalyzer; yield! externalAnalyzer |> Option.toArray |] + |> Array.collect (function + | NotificationEvent.AnalyzerMessage(diags, _, _) -> diags |> Array.map analyzersToDiagnostic + | NotificationEvent.UnnecessaryParentheses(_, ranges, _) -> + ranges |> Array.map unnecessaryParenthesesToDiagnostic + | NotificationEvent.UnusedOpens(_, ranges, _) -> ranges |> Array.map unusedOpensToDiagnostic + | NotificationEvent.UnusedDeclarations(_, ranges, _) -> ranges |> Array.map unusedDeclarationsToDiagnostic + | NotificationEvent.SimplifyNames(_, ranges, _) -> ranges |> Array.map simplifyNamesToDiagnostic + | _ -> [||]) + + let diags = [| yield! fcsDiags; yield! analyzerDiags |] + return diags + } + interface IDisposable with member this.Dispose() = diff --git a/src/FsAutoComplete/LspServers/AdaptiveServerState.fsi b/src/FsAutoComplete/LspServers/AdaptiveServerState.fsi index 8fecbcfd1..2d3fc6e5f 100644 --- a/src/FsAutoComplete/LspServers/AdaptiveServerState.fsi +++ b/src/FsAutoComplete/LspServers/AdaptiveServerState.fsi @@ -129,4 +129,6 @@ type AdaptiveState = /// See LSP Spec on WorkDoneProgress Cancel for more information. /// member CancelServerProgress: progressToken: ProgressToken -> unit + + member GetDiagnostics: file: string -> Async> interface IDisposable diff --git a/test/FsAutoComplete.Tests.Lsp/CodeLensTests.fs b/test/FsAutoComplete.Tests.Lsp/CodeLensTests.fs index 8d77dd715..223339398 100644 --- a/test/FsAutoComplete.Tests.Lsp/CodeLensTests.fs +++ b/test/FsAutoComplete.Tests.Lsp/CodeLensTests.fs @@ -16,7 +16,8 @@ open Helpers.Expecto.ShadowedTimeouts open System.IO module private CodeLens = - let examples = Path.Combine(__SOURCE_DIRECTORY__, "TestCases", "CodeLensProjectTests") + let examples = + Path.Combine(__SOURCE_DIRECTORY__, "TestCases", "CodeLensProjectTests") module CodeLensPositionStaysAccurate = let dir = examples "CodeLens_position_stays_accurate" @@ -64,164 +65,181 @@ module private CodeLens = let projectBasedTests state = - testList "ProjectBased" [ - serverTestList ("CodeLensPositionStaysAccurate") state defaultConfigDto (Some CodeLens.CodeLensPositionStaysAccurate.dir) (fun server -> [ - - testCaseAsync "can show codelens after adding newlines to code" - <| (asyncResult { - let program = CodeLens.CodeLensPositionStaysAccurate.programFile - let! (doc, _diags) = Server.openDocument program server - - let! unresolved = CodeLens.getLenses doc - let! resolved = CodeLens.getResolvedLenses doc unresolved - - let references = - resolved - |> List.filter (fun lens -> lens.Command |>Option.exists (fun c -> c.Title.EndsWith "References")) - |> List.sortBy (fun lens -> lens.Range.Start.Line) - - Expect.hasLength references 2 "should have a reference lens" - - let lens1 = references.[0] - let lens1Range : Range = { - Start = { Line = 1u; Character = 6u } - End = { Line = 1u; Character = 20u } - } - - Expect.equal lens1.Range lens1Range "Lens 1 should be at 1:6-1:20" - - let lens2 = references.[1] - let lens2Range : Range = { - Start = { Line = 3u; Character = 6u } - End = { Line = 3u; Character = 25u } - } - - Expect.equal lens2.Range lens2Range "Lens 2 should be at 3:6-3:25" - - do! doc.Server.Server.TextDocumentDidChange({ - TextDocument = doc.VersionedTextDocumentIdentifier - ContentChanges = [| U2.C1 { - Range = { Start = { Line = 2u; Character = 0u }; End = { Line = 2u; Character = 0u }; } - RangeLength = None - Text = "\n\n" - } |] - }) + testList + "ProjectBased" + [ serverTestList + ("CodeLensPositionStaysAccurate") + state + defaultConfigDto + (Some CodeLens.CodeLensPositionStaysAccurate.dir) + (fun server -> + [ + + testCaseAsync "can show codelens after adding newlines to code" + <| (asyncResult { + let program = CodeLens.CodeLensPositionStaysAccurate.programFile + let! (doc, _diags) = Server.openDocument program server + + let! unresolved = CodeLens.getLenses doc + let! resolved = CodeLens.getResolvedLenses doc unresolved + + let references = + resolved + |> List.filter (fun lens -> lens.Command |> Option.exists (fun c -> c.Title.EndsWith "References")) + |> List.sortBy (fun lens -> lens.Range.Start.Line) + + Expect.hasLength references 2 "should have a reference lens" + + let lens1 = references.[0] + + let lens1Range: Range = + { Start = { Line = 1u; Character = 6u } + End = { Line = 1u; Character = 20u } } + + Expect.equal lens1.Range lens1Range "Lens 1 should be at 1:6-1:20" + + let lens2 = references.[1] - let! nextLens = CodeLens.getLenses doc - let! resolvedNextLens = CodeLens.getResolvedLenses doc nextLens + let lens2Range: Range = + { Start = { Line = 3u; Character = 6u } + End = { Line = 3u; Character = 25u } } - let references = - resolvedNextLens - |> List.filter (fun lens -> lens.Command |>Option.exists (fun c -> c.Title.EndsWith "References")) - |> List.sortBy (fun lens -> lens.Range.Start.Line) + Expect.equal lens2.Range lens2Range "Lens 2 should be at 3:6-3:25" - let lens1 = references.[0] - let lens1Range : Range = { - Start = { Line = 1u; Character = 6u } - End = { Line = 1u; Character = 20u } - } + do! + doc.Server.Server.TextDocumentDidChange( + { TextDocument = doc.VersionedTextDocumentIdentifier + ContentChanges = + [| U2.C1 + { Range = + { Start = { Line = 2u; Character = 0u } + End = { Line = 2u; Character = 0u } } + RangeLength = None + Text = "\n\n" } |] } + ) - Expect.equal lens1.Range lens1Range "Lens 1 should be at 1:6-1:20" + let! nextLens = CodeLens.getLenses doc + let! resolvedNextLens = CodeLens.getResolvedLenses doc nextLens - let lens2 = references.[1] - let lens2Range : Range = { - Start = { Line = 5u; Character = 6u } - End = { Line = 5u; Character = 25u } - } + let references = + resolvedNextLens + |> List.filter (fun lens -> lens.Command |> Option.exists (fun c -> c.Title.EndsWith "References")) + |> List.sortBy (fun lens -> lens.Range.Start.Line) - Expect.equal lens2.Range lens2Range "Lens 2 should be at 5:6-5:25" + let lens1 = references.[0] - return () - } - |> AsyncResult.foldResult id (fun e -> failtest $"{e}" )) + let lens1Range: Range = + { Start = { Line = 1u; Character = 6u } + End = { Line = 1u; Character = 20u } } - ] - ) - ] + Expect.equal lens1.Range lens1Range "Lens 1 should be at 1:6-1:20" + + let lens2 = references.[1] + + let lens2Range: Range = + { Start = { Line = 5u; Character = 6u } + End = { Line = 5u; Character = 25u } } + + Expect.equal lens2.Range lens2Range "Lens 2 should be at 5:6-5:25" + + return () + } + |> AsyncResult.foldResult id (fun e -> failtest $"{e}")) + + ]) ] let tests state = - testList (nameof CodeLens) [ - projectBasedTests state - serverTestList "scriptTests" state defaultConfigDto None (fun server -> - [ testCaseAsync "can show codelens for type annotation" - <| CodeLens.check server """ + testList + (nameof CodeLens) + [ projectBasedTests state + serverTestList "scriptTests" state defaultConfigDto None (fun server -> + [ testCaseAsync "can show codelens for type annotation" + <| CodeLens.check server """ module X = $0let func x = x + 1$0 - """ (fun (_doc, lenses, _unresolved, _resolved) -> async { - Expect.hasLength lenses 2 "should have a type lens and a reference lens" - let typeLens = lenses[0] - Expect.equal typeLens.Command.Value.Title "int -> int" "first lens should be a type hint of int to int" - Expect.isNone typeLens.Command.Value.Arguments "No data required for type lenses" - Expect.equal typeLens.Command.Value.Command "" "No command for type lenses" }) - - testCaseAsync "can show codelens for 0 reference count" - <| CodeLens.check server """ + """ (fun (_doc, lenses, _unresolved, _resolved) -> + async { + Expect.hasLength lenses 2 "should have a type lens and a reference lens" + let typeLens = lenses[0] + Expect.equal typeLens.Command.Value.Title "int -> int" "first lens should be a type hint of int to int" + Expect.isNone typeLens.Command.Value.Arguments "No data required for type lenses" + Expect.equal typeLens.Command.Value.Command "" "No command for type lenses" + }) + + testCaseAsync "can show codelens for 0 reference count" + <| CodeLens.check server """ module X = $0let func x = x + 1$0 - """ (fun (_doc, lenses, _unresolved, _resolved) -> async { - Expect.hasLength lenses 2 "should have a type lens and a reference lens" - let referenceLens = lenses[1] - - let emptyCommand = - Some - { Title = "0 References" - Arguments = None - Command = "" } - - Expect.equal referenceLens.Command emptyCommand "There should be no command or args for zero references" }) - testCaseAsync "can show codelens for multi reference count" - <| CodeLens.check server """ + """ (fun (_doc, lenses, _unresolved, _resolved) -> + async { + Expect.hasLength lenses 2 "should have a type lens and a reference lens" + let referenceLens = lenses[1] + + let emptyCommand = + Some + { Title = "0 References" + Arguments = None + Command = "" } + + Expect.equal referenceLens.Command emptyCommand "There should be no command or args for zero references" + }) + testCaseAsync "can show codelens for multi reference count" + <| CodeLens.check server """ module X = $0let func x = x + 1$0 let doThing () = func 1 - """ (fun (doc, lenses, _unresolved, _resolved) -> async { - - - Expect.hasLength lenses 2 "should have a type lens and a reference lens" - let referenceLens = lenses[1] - Expect.isSome referenceLens.Command "There should be a command for multiple references" - let referenceCommand = referenceLens.Command.Value - Expect.equal referenceCommand.Title "1 References" "There should be a title for multiple references" - - Expect.equal - referenceCommand.Command - "fsharp.showReferences" - "There should be a command for multiple references" - - Expect.isSome referenceCommand.Arguments "There should be arguments for multiple references" - let args = referenceCommand.Arguments.Value - Expect.equal args.Length 3 "There should be 2 args" - - let filePath, triggerPos, referenceRanges = - args[0].Value(), - (args[1] :?> JObject).ToObject(), - (args[2] :?> JArray) - |> Seq.map (fun t -> (t :?> JObject).ToObject()) - |> Array.ofSeq - - Expect.equal filePath doc.Uri "File path should be the doc we're checking" - Expect.equal triggerPos { Line = 1u; Character = 6u } "Position should be 1:6" - Expect.hasLength referenceRanges 1 "There should be 1 reference range for the `func` function" - - Expect.equal - referenceRanges[0] - { Uri = doc.Uri - Range = - { Start = { Line = 3u; Character = 19u } - End = { Line = 3u; Character = 23u } } } - "Reference range should be 0:0"}) - testCaseAsync "can show reference counts for 1-character identifier" - <| CodeLens.check server """ + """ (fun (doc, lenses, _unresolved, _resolved) -> + async { + + + Expect.hasLength lenses 2 "should have a type lens and a reference lens" + let referenceLens = lenses[1] + Expect.isSome referenceLens.Command "There should be a command for multiple references" + let referenceCommand = referenceLens.Command.Value + Expect.equal referenceCommand.Title "1 References" "There should be a title for multiple references" + + Expect.equal + referenceCommand.Command + "fsharp.showReferences" + "There should be a command for multiple references" + + Expect.isSome referenceCommand.Arguments "There should be arguments for multiple references" + let args = referenceCommand.Arguments.Value + Expect.equal args.Length 3 "There should be 2 args" + + let filePath, triggerPos, referenceRanges = + args[0].Value(), + (args[1] :?> JObject).ToObject(), + (args[2] :?> JArray) + |> Seq.map (fun t -> (t :?> JObject).ToObject()) + |> Array.ofSeq + + Expect.equal filePath doc.Uri "File path should be the doc we're checking" + Expect.equal triggerPos { Line = 1u; Character = 6u } "Position should be 1:6" + Expect.hasLength referenceRanges 1 "There should be 1 reference range for the `func` function" + + Expect.equal + referenceRanges[0] + { Uri = doc.Uri + Range = + { Start = { Line = 3u; Character = 19u } + End = { Line = 3u; Character = 23u } } } + "Reference range should be 0:0" + }) + testCaseAsync "can show reference counts for 1-character identifier" + <| CodeLens.check server """ $0let f () = ""$0 - """ (fun (_doc, lenses, _unresolved, _resolved) -> async { - Expect.hasLength lenses 2 "should have a type lens and a reference lens" - let referenceLens = lenses[1] - Expect.isSome referenceLens.Command "There should be a command for multiple references" - let referenceCommand = referenceLens.Command.Value - Expect.equal referenceCommand.Title "0 References" "There should be a title for multiple references" - Expect.equal referenceCommand.Command "" "There should be no command for multiple references" - Expect.isNone referenceCommand.Arguments "There should be arguments for multiple references"}) ]) - - ] + """ (fun (_doc, lenses, _unresolved, _resolved) -> + async { + Expect.hasLength lenses 2 "should have a type lens and a reference lens" + let referenceLens = lenses[1] + Expect.isSome referenceLens.Command "There should be a command for multiple references" + let referenceCommand = referenceLens.Command.Value + Expect.equal referenceCommand.Title "0 References" "There should be a title for multiple references" + Expect.equal referenceCommand.Command "" "There should be no command for multiple references" + Expect.isNone referenceCommand.Arguments "There should be arguments for multiple references" + }) ]) + + ] diff --git a/test/FsAutoComplete.Tests.Lsp/CoreTests.fs b/test/FsAutoComplete.Tests.Lsp/CoreTests.fs index 53d8de9ae..2adecbda0 100644 --- a/test/FsAutoComplete.Tests.Lsp/CoreTests.fs +++ b/test/FsAutoComplete.Tests.Lsp/CoreTests.fs @@ -88,6 +88,17 @@ let initTests createServer = Expect.equal res.Capabilities.DocumentLinkProvider None "Document Link Provider" Expect.equal res.Capabilities.DocumentOnTypeFormattingProvider None "Document OnType Formatting Provider" + Expect.equal + res.Capabilities.DiagnosticProvider + (Some( + U2.C1 + { WorkDoneProgress = Some false + Identifier = None + InterFileDependencies = true + WorkspaceDiagnostics = false } + )) + "Diagnostic Provider" + Expect.equal res.Capabilities.DocumentRangeFormattingProvider (Some(U2.C1 true)) @@ -634,14 +645,7 @@ let tooltipTests state = // FSI hash directive hover — regression for issue #1225. // Hovering on a hash directive (e.g. #nowarn, #r, #load) should show documentation. - verifyDescription - 123u - 3u - [ "**Description**" - "" - "" - "Disables a compiler warning or warnings" - "" ] ] ] + verifyDescription 123u 3u [ "**Description**"; ""; ""; "Disables a compiler warning or warnings"; "" ] ] ] let closeTests state = // Note: clear diagnostics also implies clear caches (-> remove file & project options from State). @@ -662,7 +666,7 @@ let closeTests state = Expect.isNonEmpty diags "There should be an error" do! doc |> Document.close - let! diags = doc |> Document.waitForLatestDiagnostics (TimeSpan.FromSeconds 5.0) + let! diags = doc |> Document.waitForLatestPublishedDiagnostics (TimeSpan.FromSeconds 5.0) Expect.equal diags Array.empty "There should be a final publishDiagnostics without any diags" }) testCaseAsync @@ -672,7 +676,7 @@ let closeTests state = Expect.isNonEmpty diags "There should be an error" do! doc |> Document.close - let! diags = doc |> Document.waitForLatestDiagnostics (TimeSpan.FromSeconds 5.0) + let! diags = doc |> Document.waitForLatestPublishedDiagnostics (TimeSpan.FromSeconds 5.0) Expect.isNonEmpty diags "There should be no publishDiagnostics without any diags after close" }) testCaseAsync @@ -683,7 +687,7 @@ let closeTests state = Expect.isNonEmpty diags "There should be an error" do! doc |> Document.close - let! diags = doc |> Document.waitForLatestDiagnostics (TimeSpan.FromSeconds 5.0) + let! diags = doc |> Document.waitForLatestPublishedDiagnostics (TimeSpan.FromSeconds 5.0) Expect.isEmpty diags "There should be a final publishDiagnostics without any diags" }) @@ -694,7 +698,7 @@ let closeTests state = Expect.isNonEmpty diags "There should be an error" do! doc |> Document.close - let! diags = doc |> Document.waitForLatestDiagnostics (TimeSpan.FromSeconds 5.0) + let! diags = doc |> Document.waitForLatestPublishedDiagnostics (TimeSpan.FromSeconds 5.0) Expect.isNonEmpty diags "There should be no publishDiagnostics without any diags after close" }) testCaseAsync @@ -705,7 +709,7 @@ let closeTests state = Expect.isNonEmpty diags "There should be an error" do! doc |> Document.close - let! diags = doc |> Document.waitForLatestDiagnostics (TimeSpan.FromSeconds 5.0) + let! diags = doc |> Document.waitForLatestPublishedDiagnostics (TimeSpan.FromSeconds 5.0) Expect.isNonEmpty diags "There should be no publishDiagnostics without any diags after close" }) ]) diff --git a/test/FsAutoComplete.Tests.Lsp/EmptyFileTests.fs b/test/FsAutoComplete.Tests.Lsp/EmptyFileTests.fs index 420b3f6c2..13fdedbb7 100644 --- a/test/FsAutoComplete.Tests.Lsp/EmptyFileTests.fs +++ b/test/FsAutoComplete.Tests.Lsp/EmptyFileTests.fs @@ -83,8 +83,6 @@ let tests state = End = { Line = 0u; Character = 0u } } RangeLength = Some 0u Text = "c" } |] } - // wait for typechecking to propogate? - do! Async.Sleep 1000 let! completions = server.TextDocumentCompletion diff --git a/test/FsAutoComplete.Tests.Lsp/Expecto.OpenTelemetry.fs b/test/FsAutoComplete.Tests.Lsp/Expecto.OpenTelemetry.fs new file mode 100644 index 000000000..483143844 --- /dev/null +++ b/test/FsAutoComplete.Tests.Lsp/Expecto.OpenTelemetry.fs @@ -0,0 +1,252 @@ +namespace Expecto + +module OpenTelemetry = + open System + open System.Diagnostics + open System.Collections.Generic + open Impl + open System.Runtime.CompilerServices + + type Activity with + member inline x.SetSource + ( + ?name_space: string, + [] ?memberName: string, + [] ?path: string, + [] ?line: int + ) = + if not (isNull x) then + let name_space = + name_space + |> Option.defaultWith (fun () -> + Reflection.MethodBase.GetCurrentMethod().DeclaringType.FullName.Split("+") // F# has + in type names that refer to anonymous functions, we typically want the first named type + |> Seq.tryHead + |> Option.defaultValue "") + + if x.GetTagItem "code.namespace" = null then + x.SetTag("code.namespace", name_space) |> ignore + + if x.GetTagItem "code.function" = null then + x.SetTag("code.function", defaultArg memberName "") |> ignore + + if x.GetTagItem "code.filepath" = null then + x.SetTag("code.filepath", defaultArg path "") |> ignore + + if x.GetTagItem "code.lineno" = null then + x.SetTag("code.lineno", defaultArg line 0) |> ignore + + module internal Activity = + let inline isNotNull x = isNull x |> not + + let inline setStatus (status: ActivityStatusCode) (span: Activity) = + if isNotNull span then + span.SetStatus(status) |> ignore + + let inline setExn (e: exn) (span: Activity) = + if isNotNull span then + let tags = + ActivityTagsCollection( + seq { + KeyValuePair("exception.type", box (e.GetType().Name)) + KeyValuePair("exception.stacktrace", box (e.ToString())) + + if not <| String.IsNullOrEmpty(e.Message) then + KeyValuePair("exception.message", box e.Message) + } + ) + + ActivityEvent("exception", tags = tags) |> span.AddEvent |> ignore + + let inline setExnMarkFailed (e: exn) (span: Activity) = + if isNotNull span then + setExn e span + span.SetStatus(ActivityStatusCode.Error, e.Message) |> ignore + + let setSourceLocation (sourceLoc: SourceLocation) (span: Activity) = + if isNotNull span && sourceLoc <> SourceLocation.empty then + span.SetTag("code.lineno", sourceLoc.lineNumber) |> ignore + span.SetTag("code.filepath", sourceLoc.sourcePath) |> ignore + + let inline addOutcome (result: TestResult) (span: Activity) = + if isNotNull span then + let status = + match result with + | Passed -> "Passed" + | Ignored _ -> "Ignored" + | Failed _ -> "Failed" + | Error _ -> "Error" + + span.SetTag("test.result.status", status) |> ignore + span.SetTag("test.result.message", result) |> ignore + + let inline start (span: Activity) = + if isNotNull span then + span.Start() |> ignore + + span + + let inline stop (span: Activity) = + if isNotNull span then + span.Stop() |> ignore + + let inline setEndTimeNow (span: Activity) = + if isNotNull span then + span.SetEndTime(DateTime.UtcNow) |> ignore + + let inline createActivity (name: string) (source: ActivitySource) = + if isNotNull source then + source.CreateActivity(name, ActivityKind.Internal) + else + null + + open Activity + open System.Runtime.ExceptionServices + + let inline internal reraiseAnywhere<'a> (e: exn) : 'a = + ExceptionDispatchInfo.Capture(e).Throw() + Unchecked.defaultof<'a> + + module TestResult = + let ofException (e: Exception) : TestResult = + match e with + | :? AssertException as e -> + let msg = + "\n" + + e.Message + + "\n" + + (e.StackTrace.Split('\n') + |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) + |> Seq.truncate 5 + |> String.concat "\n") + + Failed msg + + | :? FailedException as e -> Failed("\n" + e.Message) + | :? IgnoreException as e -> Ignored e.Message + | :? AggregateException as e when e.InnerExceptions.Count = 1 -> + if e.InnerException :? IgnoreException then + Ignored e.InnerException.Message + else + Error e.InnerException + | e -> Error e + + + let addExceptionOutcomeToSpan (span: Activity) (e: Exception) = + let testResult = TestResult.ofException e + + addOutcome testResult span + + match testResult with + | Ignored _ -> setExn e span + | _ -> setExnMarkFailed e span + + let wrapCodeWithLazySpan + (activitySource: ActivitySource) + (testName: string) + (sourceLoc: SourceLocation) + (test: TestCode) + = + let createAndConfigureSpan () = + let previous = Activity.Current + Activity.Current <- null + let span = activitySource |> createActivity testName + span |> setSourceLocation sourceLoc + span |> start |> ignore + + if isNull span then + Activity.Current <- previous + + span, previous + + let disposeAndRestore (span: Activity) previous = + if isNotNull span then + span.Dispose() + + Activity.Current <- previous + + let inline handleSuccess span = + setEndTimeNow span + addOutcome Passed span + setStatus ActivityStatusCode.Ok span + + let inline handleFailure span e = + setEndTimeNow span + addExceptionOutcomeToSpan span e + reraiseAnywhere e + + match test with + | Sync test -> + TestCode.Sync(fun () -> + let span, previous = createAndConfigureSpan () + + try + try + test () + handleSuccess span + with e -> + handleFailure span e + finally + disposeAndRestore span previous) + + | Async test -> + TestCode.Async( + async { + let span, previous = createAndConfigureSpan () + + try + try + do! test + handleSuccess span + with e -> + handleFailure span e + finally + disposeAndRestore span previous + } + ) + + | AsyncFsCheck(testConfig, stressConfig, test) -> + TestCode.AsyncFsCheck( + testConfig, + stressConfig, + fun fsCheckConfig -> + async { + let span, previous = createAndConfigureSpan () + + try + try + do! test fsCheckConfig + handleSuccess span + with e -> + handleFailure span e + finally + disposeAndRestore span previous + } + ) + + | SyncWithCancel test -> + TestCode.SyncWithCancel(fun ct -> + let span, previous = createAndConfigureSpan () + + try + try + test ct + handleSuccess span + with e -> + handleFailure span e + finally + disposeAndRestore span previous) + + let addOpenTelemetry_SpanPerTest (config: ExpectoConfig) (activitySource: ActivitySource) (rootTest: Test) : Test = + rootTest + |> Test.toTestCodeList + |> List.map (fun test -> + let testName = config.joinWith.format test.name + let sourceLoc = config.locate test.test + + { test with + test = wrapCodeWithLazySpan activitySource testName sourceLoc test.test }) + |> Test.fromFlatTests config.joinWith.asString + + let serviceName = "FsAutoComplete.Tests.Lsp" + + let source = new ActivitySource(serviceName) diff --git a/test/FsAutoComplete.Tests.Lsp/ExtensionsTests.fs b/test/FsAutoComplete.Tests.Lsp/ExtensionsTests.fs index af3a6a27f..373e2aafb 100644 --- a/test/FsAutoComplete.Tests.Lsp/ExtensionsTests.fs +++ b/test/FsAutoComplete.Tests.Lsp/ExtensionsTests.fs @@ -351,7 +351,6 @@ let analyzerTests state = let! (server, events) = serverInitialize path analyzerEnabledConfig state let scriptPath = Path.Combine(path, "Script.fsx") - do! Async.Sleep(TimeSpan.FromSeconds 5.) do! waitForWorkspaceFinishedParsing events do! server.TextDocumentDidOpen { TextDocument = loadDocument scriptPath } return server, events, path, scriptPath diff --git a/test/FsAutoComplete.Tests.Lsp/FsAutoComplete.Tests.Lsp.fsproj b/test/FsAutoComplete.Tests.Lsp/FsAutoComplete.Tests.Lsp.fsproj index ee58fe033..45a35ec39 100644 --- a/test/FsAutoComplete.Tests.Lsp/FsAutoComplete.Tests.Lsp.fsproj +++ b/test/FsAutoComplete.Tests.Lsp/FsAutoComplete.Tests.Lsp.fsproj @@ -35,6 +35,8 @@ + + diff --git a/test/FsAutoComplete.Tests.Lsp/Helpers.fs b/test/FsAutoComplete.Tests.Lsp/Helpers.fs index a180e900d..667be3a1e 100644 --- a/test/FsAutoComplete.Tests.Lsp/Helpers.fs +++ b/test/FsAutoComplete.Tests.Lsp/Helpers.fs @@ -637,7 +637,6 @@ let parseProject projectFilePath (server: IFSharpLspServer) = let projectName = Path.GetFileNameWithoutExtension projectFilePath let! result = server.FSharpProject projectParams - do! Async.Sleep(TimeSpan.FromSeconds 3.) logger.Value.Debug("{project} parse result: {result}", projectName, result) } diff --git a/test/FsAutoComplete.Tests.Lsp/OpenTelemetry.Exporter.fs b/test/FsAutoComplete.Tests.Lsp/OpenTelemetry.Exporter.fs new file mode 100644 index 000000000..4128c4f24 --- /dev/null +++ b/test/FsAutoComplete.Tests.Lsp/OpenTelemetry.Exporter.fs @@ -0,0 +1,391 @@ +module OpenTelemetry.Exporter.OtlpFile + +open System +open System.Diagnostics +open System.IO +open System.Text.Json +open System.Text.Json.Serialization +open OpenTelemetry + +[] +module OtlpJson = + + let toUnixNano (dt: DateTime) : string = + let unixEpoch = DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc) + let ticks = (dt.ToUniversalTime() - unixEpoch).Ticks + let nanos = ticks * 100L // 1 tick = 100 nanoseconds + string nanos + + // OTLP JSON requires hexadecimal trace and span identifiers. + let traceIdToHex (traceId: ActivityTraceId) : string = traceId.ToHexString().ToLowerInvariant() + + let spanIdToHex (spanId: ActivitySpanId) : string = spanId.ToHexString().ToLowerInvariant() + + type KeyValue = + { [] + Key: string + [] + Value: AnyValue } + + and AnyValue = + { [] + [] + StringValue: string + [] + [] + IntValue: string + [] + [] + BoolValue: bool + [] + [] + DoubleValue: float } + + let toAnyValue (value: obj) : AnyValue = + match value with + | :? string as s -> + { StringValue = s + IntValue = null + BoolValue = false + DoubleValue = 0.0 } + | :? int as i -> + { StringValue = null + IntValue = string i + BoolValue = false + DoubleValue = 0.0 } + | :? int64 as i -> + { StringValue = null + IntValue = string i + BoolValue = false + DoubleValue = 0.0 } + | :? bool as b -> + { StringValue = null + IntValue = null + BoolValue = b + DoubleValue = 0.0 } + | :? float as f -> + { StringValue = null + IntValue = null + BoolValue = false + DoubleValue = f } + | _ -> + { StringValue = (if isNull value then "" else value.ToString()) + IntValue = null + BoolValue = false + DoubleValue = 0.0 } + + type SpanEvent = + { [] + TimeUnixNano: string + [] + Name: string + [] + Attributes: KeyValue[] } + + type SpanStatus = + { [] + Code: int + [] + [] + Message: string } + + type OtlpSpan = + { [] + TraceId: string + [] + SpanId: string + [] + [] + ParentSpanId: string + [] + Name: string + [] + Kind: int + [] + StartTimeUnixNano: string + [] + EndTimeUnixNano: string + [] + Attributes: KeyValue[] + [] + Events: SpanEvent[] + [] + Status: SpanStatus } + + type InstrumentationScope = + { [] + Name: string + [] + [] + Version: string } + + type ScopeSpans = + { [] + Scope: InstrumentationScope + [] + Spans: OtlpSpan[] } + + type ResourceAttributes = + { [] + Attributes: KeyValue[] } + + type ResourceSpans = + { [] + Resource: ResourceAttributes + [] + ScopeSpans: ScopeSpans[] } + + type TracesData = + { [] + ResourceSpans: ResourceSpans[] } + + let fromActivity (activity: Activity) : OtlpSpan = + let attributes = + activity.Tags + |> Seq.map (fun kvp -> + { Key = kvp.Key + Value = toAnyValue kvp.Value }) + |> Seq.toArray + + let events = + activity.Events + |> Seq.map (fun evt -> + { TimeUnixNano = toUnixNano evt.Timestamp.UtcDateTime + Name = evt.Name + Attributes = + evt.Tags + |> Seq.map (fun kvp -> + { Key = kvp.Key + Value = toAnyValue kvp.Value }) + |> Seq.toArray }) + |> Seq.toArray + + let statusCode = + match activity.Status with + | ActivityStatusCode.Error -> 2 + | ActivityStatusCode.Ok -> 1 + | _ -> 0 + + let parentSpanId = + if activity.ParentSpanId.ToHexString() = "0000000000000000" then + null + else + spanIdToHex activity.ParentSpanId + + let spanKind = + match activity.Kind with + | ActivityKind.Internal -> 1 + | ActivityKind.Server -> 2 + | ActivityKind.Client -> 3 + | ActivityKind.Producer -> 4 + | ActivityKind.Consumer -> 5 + | _ -> 0 + + { TraceId = traceIdToHex activity.TraceId + SpanId = spanIdToHex activity.SpanId + ParentSpanId = parentSpanId + Name = activity.DisplayName + Kind = spanKind + StartTimeUnixNano = toUnixNano activity.StartTimeUtc + EndTimeUnixNano = toUnixNano (activity.StartTimeUtc + activity.Duration) + Attributes = attributes + Events = events + Status = + { Code = statusCode + Message = activity.StatusDescription } } + + let toTracesData (serviceName: string) (serviceVersion: string) (spans: OtlpSpan seq) : TracesData = + { ResourceSpans = + [| { Resource = + { Attributes = + [| { Key = "service.name" + Value = toAnyValue serviceName } + { Key = "service.version" + Value = toAnyValue serviceVersion } |] } + ScopeSpans = + [| { Scope = + { Name = serviceName + Version = serviceVersion } + Spans = spans |> Seq.toArray } |] } |] } + + +type OtlpFileExporterOptions() = + member val OutputDirectory: string = "traces" with get, set + + member val ServiceName: string = "unknown" with get, set + + member val ServiceVersion: string = "0.0.0" with get, set + + member val Filter: (Activity -> bool) option = None with get, set + + member val FilePrefix: string = "traces" with get, set + + member val PrettyPrint: bool = true with get, set + +let private writeSpansToFile (options: OtlpFileExporterOptions) (spans: OtlpJson.OtlpSpan[]) = + if Array.isEmpty spans then + None + else + Directory.CreateDirectory(options.OutputDirectory) |> ignore + + let timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss_fff") + + let filename = + Path.Combine(options.OutputDirectory, $"{options.FilePrefix}_{timestamp}_{Environment.ProcessId}.otlp.json") + + let jsonOptions = JsonSerializerOptions(WriteIndented = options.PrettyPrint) + + let tracesData = + OtlpJson.toTracesData options.ServiceName options.ServiceVersion spans + + let json = JsonSerializer.Serialize(tracesData, jsonOptions) + File.WriteAllText(filename, json) + Some(filename, spans.Length) + + +type OtlpFileExportProcessor(options: OtlpFileExporterOptions) = + inherit BaseProcessor() + + let collectedSpans = Collections.Concurrent.ConcurrentBag() + let mutable exportedFileCount = 0 + + let shouldExport (activity: Activity) = + match options.Filter with + | Some filter -> filter activity + | None -> true + + let writeToFile () = + let result = collectedSpans.ToArray() |> writeSpansToFile options + + if Option.isSome result then + exportedFileCount <- exportedFileCount + 1 + + while not (collectedSpans.IsEmpty) do + collectedSpans.TryTake() |> ignore + + result + + override _.OnEnd(activity: Activity) = + if shouldExport activity then + collectedSpans.Add(OtlpJson.fromActivity activity) + + base.OnEnd(activity) + + member _.CollectedSpanCount = collectedSpans.Count + + member _.ExportedFileCount = exportedFileCount + + member _.WriteToFile() = writeToFile () + + interface IDisposable with + member this.Dispose() = + this.WriteToFile() |> ignore + base.Dispose() + + +type FailedTestOtlpFileExportProcessor(options: OtlpFileExporterOptions) = + inherit BaseProcessor() + + let options = + OtlpFileExporterOptions( + OutputDirectory = options.OutputDirectory, + ServiceName = options.ServiceName, + ServiceVersion = options.ServiceVersion, + FilePrefix = + (if String.IsNullOrEmpty(options.FilePrefix) || options.FilePrefix = "traces" then + "failed_tests" + else + options.FilePrefix), + PrettyPrint = options.PrettyPrint + ) + + let traces = + Collections.Concurrent.ConcurrentDictionary>() + + let failedTraceIds = + Collections.Concurrent.ConcurrentDictionary() + + let completedSuccessfulTraceIds = + Collections.Concurrent.ConcurrentDictionary() + + let mutable exportedFileCount = 0 + + let testStatus (activity: Activity) = + match activity.GetTagItem("test.result.status") with + | :? string as status -> Some status + | _ -> None + + let isRecorded (activity: Activity) = + activity.ActivityTraceFlags &&& ActivityTraceFlags.Recorded = ActivityTraceFlags.Recorded + + let writeToFile () = + let failedIds = failedTraceIds.Keys |> Seq.toArray + + let spans = + failedIds + |> Array.collect (fun traceId -> + match traces.TryGetValue traceId with + | true, spans -> spans.ToArray() + | false, _ -> [||]) + + let result = spans |> writeSpansToFile options + + if Option.isSome result then + exportedFileCount <- exportedFileCount + 1 + + for traceId in failedIds do + traces.TryRemove traceId |> ignore + failedTraceIds.TryRemove traceId |> ignore + + result |> Option.map (fun (filename, _) -> filename, failedIds.Length) + + override _.OnEnd(activity: Activity) = + if isRecorded activity then + match testStatus activity with + | Some("Passed" | "Ignored") -> + completedSuccessfulTraceIds.TryAdd(activity.TraceId, ()) |> ignore + traces.TryRemove activity.TraceId |> ignore + | status when not (completedSuccessfulTraceIds.ContainsKey activity.TraceId) -> + traces + .GetOrAdd(activity.TraceId, fun _ -> Collections.Concurrent.ConcurrentBag()) + .Add(OtlpJson.fromActivity activity) + + match status with + | Some("Failed" | "Error") -> failedTraceIds.TryAdd(activity.TraceId, ()) |> ignore + | _ -> () + | _ -> () + + base.OnEnd activity + + member _.CollectedSpanCount = traces.Values |> Seq.sumBy _.Count + member _.ExportedFileCount = exportedFileCount + member _.WriteToFile() = writeToFile () + + interface IDisposable with + member this.Dispose() = + this.WriteToFile() |> ignore + base.Dispose() + + +[] +module TracerProviderBuilderExtensions = + open OpenTelemetry.Trace + + type TracerProviderBuilder with + + member this.AddOtlpFileExporter() = + let options = OtlpFileExporterOptions() + let processor = new OtlpFileExportProcessor(options) + this.AddProcessor(processor), processor + + member this.AddOtlpFileExporter(configure: OtlpFileExporterOptions -> unit) = + let options = OtlpFileExporterOptions() + configure options + let processor = new OtlpFileExportProcessor(options) + this.AddProcessor(processor), processor + + member this.AddFailedTestOtlpFileExporter(configure: OtlpFileExporterOptions -> unit) = + let options = OtlpFileExporterOptions() + configure options + let processor = new FailedTestOtlpFileExportProcessor(options) + this.AddProcessor(processor), processor diff --git a/test/FsAutoComplete.Tests.Lsp/OpenTelemetryTests.fs b/test/FsAutoComplete.Tests.Lsp/OpenTelemetryTests.fs new file mode 100644 index 000000000..cca064e41 --- /dev/null +++ b/test/FsAutoComplete.Tests.Lsp/OpenTelemetryTests.fs @@ -0,0 +1,160 @@ +module FsAutoComplete.Tests.OpenTelemetryTests + +open System +open System.Collections.Generic +open System.Diagnostics +open System.IO +open System.Text.Json +open Expecto +open Expecto.Impl +open OpenTelemetry +open OpenTelemetry.Exporter.OtlpFile +open OpenTelemetry.Trace + +type private CollectingProcessor(spans: ResizeArray) = + inherit BaseProcessor() + + override _.OnEnd(activity: Activity) = + spans.Add activity + base.OnEnd activity + +let private withTempDirectory f = + let directory = + Path.Combine(Path.GetTempPath(), "FsAutoComplete.Tests", Guid.NewGuid().ToString()) + + Directory.CreateDirectory directory |> ignore + + try + f directory + finally + Directory.Delete(directory, true) + +let private withoutCurrentActivity f = + let previousActivity = Activity.Current + Activity.Current <- null + + try + f () + finally + Activity.Current <- previousActivity + +let private spanNames (path: string) = + use json = JsonDocument.Parse(File.ReadAllText path) + + [| for resourceSpans in json.RootElement.GetProperty("resourceSpans").EnumerateArray() do + for scopeSpans in resourceSpans.GetProperty("scopeSpans").EnumerateArray() do + for span in scopeSpans.GetProperty("spans").EnumerateArray() do + span.GetProperty("name").GetString() |] + +let private spanKinds (path: string) = + use json = JsonDocument.Parse(File.ReadAllText path) + + [| for resourceSpans in json.RootElement.GetProperty("resourceSpans").EnumerateArray() do + for scopeSpans in resourceSpans.GetProperty("scopeSpans").EnumerateArray() do + for span in scopeSpans.GetProperty("spans").EnumerateArray() do + span.GetProperty("kind").GetInt32() |] + +let tests = + testList + "OpenTelemetry" + [ testCase "test spans are independent roots with exception details" (fun () -> + use source = new ActivitySource($"FsAutoComplete.Tests.%O{Guid.NewGuid()}") + let stoppedSpans = ResizeArray() + + use provider = + Sdk + .CreateTracerProviderBuilder() + .AddSource(source.Name) + .AddProcessor(new CollectingProcessor(stoppedSpans)) + .Build() + + use outer = source.StartActivity("outer") + + let testCode = + Expecto.OpenTelemetry.wrapCodeWithLazySpan + source + "failed test" + SourceLocation.empty + (TestCode.Sync(fun () -> raise (InvalidOperationException "boom"))) + + match testCode with + | TestCode.Sync run -> + Expect.throwsT run "The wrapper must preserve the test exception" + | _ -> failtest "Expected synchronous test code" + + let testSpan = + stoppedSpans |> Seq.find (fun span -> span.DisplayName = "failed test") + + Expect.notEqual testSpan.TraceId outer.TraceId "Each test must start a separate trace" + Expect.equal testSpan.ParentSpanId (ActivitySpanId()) "A test span must be a trace root" + Expect.equal testSpan.Status ActivityStatusCode.Error "A failed test must set error status" + Expect.equal testSpan.StatusDescription "boom" "A failed test must include the exception message" + + Expect.exists + testSpan.Events + (fun event -> event.Name = "exception") + "A failed test must include an exception event") + + testCase "failed test export includes its child spans only" (fun () -> + withTempDirectory (fun directory -> + withoutCurrentActivity (fun () -> + use source = new ActivitySource($"FsAutoComplete.Tests.%O{Guid.NewGuid()}") + + let builder, exporter = + Sdk + .CreateTracerProviderBuilder() + .AddSource(source.Name) + .AddFailedTestOtlpFileExporter(fun options -> + options.OutputDirectory <- directory + options.ServiceName <- "tests" + options.ServiceVersion <- "1.0.0") + + use provider = builder.Build() + + use failedTest = + source.StartActivity("failed test", ActivityKind.Internal, ActivityContext()) + + do + use child = source.StartActivity("failed child") + () + + failedTest.SetTag("test.result.status", "Failed") |> ignore + failedTest.SetStatus(ActivityStatusCode.Error) |> ignore + failedTest.Stop() + + use passedTest = + source.StartActivity("passed test", ActivityKind.Internal, ActivityContext()) + + do + use child = source.StartActivity("passed child") + () + + passedTest.SetTag("test.result.status", "Passed") |> ignore + passedTest.SetStatus(ActivityStatusCode.Ok) |> ignore + passedTest.Stop() + + provider.ForceFlush() |> ignore + + let path, failedTestCount = + exporter.WriteToFile() + |> Option.defaultWith (fun () -> failtest "Expected a failed test trace file") + + let names = spanNames path + let kinds = spanKinds path + + Expect.equal failedTestCount 1 "The exporter must report the failed test count" + + Expect.stringContains + (Path.GetFileName path) + $"_{Environment.ProcessId}.otlp.json" + "The trace filename must identify its test process" + + Expect.contains names "failed test" "The export must contain the failed test span" + Expect.contains names "failed child" "The export must contain child spans from the failed test" + Expect.isFalse (names |> Array.contains "passed test") "The export must exclude passed test spans" + + Expect.isFalse + (names |> Array.contains "passed child") + "The export must exclude child spans from passed tests" + + Expect.all kinds ((=) 1) "Internal activities must use the OTLP internal span kind"))) ] diff --git a/test/FsAutoComplete.Tests.Lsp/Program.fs b/test/FsAutoComplete.Tests.Lsp/Program.fs index 38044b703..a6c0c245e 100644 --- a/test/FsAutoComplete.Tests.Lsp/Program.fs +++ b/test/FsAutoComplete.Tests.Lsp/Program.fs @@ -21,9 +21,107 @@ open System.IO open FsAutoComplete open Helpers open FsToolkit.ErrorHandling +open System.Diagnostics +open OpenTelemetry.Resources +open OpenTelemetry +open OpenTelemetry.Exporter +open OpenTelemetry.Exporter.OtlpFile +open OpenTelemetry.Trace Expect.defaultDiffPrinter <- Diff.colourisedDiff +let resourceBuilder version = + ResourceBuilder.CreateDefault().AddService(serviceName = serviceName, serviceVersion = version) + +let isCI = Environment.GetEnvironmentVariable("CI") = "true" + +let failedTracesDirectory = + let dir = + Environment.GetEnvironmentVariable("FAILED_TRACES_DIR") + |> Option.ofObj + |> Option.defaultValue "failed_traces" + + Path.GetFullPath(dir) + +type SpanFilter(filter: Activity -> bool) = + inherit BaseProcessor() + + override x.OnEnd(span: Activity) : unit = + if filter span then + span.ActivityTraceFlags <- span.ActivityTraceFlags &&& (~~~ActivityTraceFlags.Recorded) + else + base.OnEnd(span: Activity) + +type TracerProviderBuilder with + member x.AddSpanFilter(filter: Activity -> bool) = x.AddProcessor(new SpanFilter(filter)) + +let private currentTraceProvider: TracerProvider option ref = ref None + +let private currentFailedTestExporter: FailedTestOtlpFileExportProcessor option ref = + ref None + +let private tracesAlreadyWritten = ref false + +let private writeFailedTraces () = + if not tracesAlreadyWritten.Value then + tracesAlreadyWritten.Value <- true + + match currentFailedTestExporter.Value with + | Some exporter -> + let traceResult = exporter.WriteToFile() + + match traceResult with + | Some(file, count) -> + if not (Directory.Exists failedTracesDirectory) then + Directory.CreateDirectory failedTracesDirectory |> ignore + + let summaryFile = + Path.Combine(failedTracesDirectory, $"summary-{Environment.ProcessId}.txt") + + File.WriteAllText(summaryFile, $"Failed test traces: {file}\nFailed test count: {count}") + printfn $"::error::Found {count} failed test(s). Traces written to {file}" + | None -> printfn "No failed tests – no traces written" + | None -> () + +let private flushTraceProvider () = + match currentTraceProvider.Value with + | Some provider -> provider.ForceFlush(3000) |> ignore + | None -> () + +// YoloDev.Expecto.TestSdk can initialize tests without calling main. +do + let version = FsAutoComplete.Utils.Version.info().Version + + let baseBuilder = + Sdk + .CreateTracerProviderBuilder() + .AddSource(FsAutoComplete.Utils.Tracing.serviceName, Tracing.fscServiceName, serviceName) + .SetResourceBuilder(resourceBuilder version) + .AddSpanFilter((fun span -> span.DisplayName.Contains "DiagnosticsLogger")) // DiagnosticsLogger.StackGuard.Guard is too noisy + + if isCI then + printfn $"Running in CI mode – failed test traces will be written to: {failedTracesDirectory}" + + let builder, exporter = + baseBuilder.AddFailedTestOtlpFileExporter(fun opts -> + opts.OutputDirectory <- failedTracesDirectory + opts.ServiceName <- serviceName + opts.ServiceVersion <- version) + + currentTraceProvider.Value <- Some(builder.Build()) + currentFailedTestExporter.Value <- Some exporter + + AppDomain.CurrentDomain.ProcessExit.Add(fun _ -> + writeFailedTraces () + flushTraceProvider ()) + else + let otlpEndpoint = Environment.GetEnvironmentVariable "OTEL_EXPORTER_OTLP_ENDPOINT" + + if not (String.IsNullOrEmpty otlpEndpoint) then + currentTraceProvider.Value <- Some(baseBuilder.AddOtlpExporter().Build()) + + AppDomain.CurrentDomain.ProcessExit.Add(fun _ -> flushTraceProvider ()) + let testTimeout = Environment.GetEnvironmentVariable "TEST_TIMEOUT_MINUTES" @@ -90,7 +188,11 @@ let selectTestGroups groups = groups |> List.choose (fun (shard, test) -> if shard = selectedShard then Some test else None) +let otelTests = + OpenTelemetry.addOpenTelemetry_SpanPerTest Expecto.Impl.ExpectoConfig.defaultConfig source + let lspTests = + testSequenced <| testList "lsp" @@ -181,40 +283,22 @@ let generalTests = TipFormatterTests.allTests FcsInvariantTests.tests FsProjEditorTests.allTests + OpenTelemetryTests.tests decompilerTests ] [] let tests = - match testShard with - | None -> testList "FSAC" [ generalTests; lspTests; SnapshotTests.snapshotTests loaders toolsPath ] - | Some 1 -> testList "FSAC" [ generalTests; lspTests ] - | Some 4 -> testList "FSAC" [ lspTests; SnapshotTests.snapshotTests loaders toolsPath ] - | Some _ -> testList "FSAC" [ lspTests ] + (match testShard with + | None -> testList "FSAC" [ generalTests; lspTests; SnapshotTests.snapshotTests loaders toolsPath ] + | Some 1 -> testList "FSAC" [ generalTests; lspTests ] + | Some 4 -> testList "FSAC" [ lspTests; SnapshotTests.snapshotTests loaders toolsPath ] + | Some _ -> testList "FSAC" [ lspTests ]) + |> otelTests -open OpenTelemetry -open OpenTelemetry.Resources -open OpenTelemetry.Trace -open OpenTelemetry.Logs -open OpenTelemetry.Metrics -open System.Diagnostics open FsAutoComplete.Telemetry [] let main args = - let serviceName = "FsAutoComplete.Tests.Lsp" - - use traceProvider = - let version = FsAutoComplete.Utils.Version.info().Version - - Sdk - .CreateTracerProviderBuilder() - .AddSource(FsAutoComplete.Utils.Tracing.serviceName, Tracing.fscServiceName, serviceName) - .SetResourceBuilder( - ResourceBuilder.CreateDefault().AddService(serviceName = serviceName, serviceVersion = version) - ) - .AddOtlpExporter() - .Build() - let outputTemplate = "[{Timestamp:HH:mm:ss} {Level:u3}] [{SourceContext}] {Message:lj}{NewLine}{Exception}" @@ -316,7 +400,6 @@ let main args = let fixedUpArgs = args |> Array.except argsToRemove let cts = new CancellationTokenSource(testTimeout) - use activitySource = new ActivitySource(serviceName) let cliArgs = [ CLIArguments.Printer(Expecto.Impl.TestPrinters.summaryWithLocationPrinter defaultConfig.printer) @@ -324,5 +407,10 @@ let main args = CLIArguments.Parallel ] // let trace = traceProvider.GetTracer("FsAutoComplete.Tests.Lsp") // use span = trace.StartActiveSpan("runTests", SpanKind.Internal) - use span = activitySource.StartActivity("runTests") - runTestsWithCLIArgsAndCancel cts.Token cliArgs fixedUpArgs tests + use span = source.StartActivity("runTests") + let result = runTestsWithCLIArgsAndCancel cts.Token cliArgs fixedUpArgs tests + + writeFailedTraces () + flushTraceProvider () + + result diff --git a/test/FsAutoComplete.Tests.Lsp/ScriptTests.fs b/test/FsAutoComplete.Tests.Lsp/ScriptTests.fs index 7d1f585b4..fea1654c0 100644 --- a/test/FsAutoComplete.Tests.Lsp/ScriptTests.fs +++ b/test/FsAutoComplete.Tests.Lsp/ScriptTests.fs @@ -189,8 +189,6 @@ let scriptProjectOptionsCacheTests state = (async { let! server, _events, _workingDir, testFilePath, allOpts = server do! server.TextDocumentDidOpen { TextDocument = loadDocument testFilePath } - do! Async.Sleep(TimeSpan.FromSeconds 3.) do! server.TextDocumentDidOpen { TextDocument = loadDocument testFilePath } - do! Async.Sleep(TimeSpan.FromSeconds 3.) Expect.hasLength allOpts 1 "should only have one event" }) ] ] diff --git a/test/FsAutoComplete.Tests.Lsp/Utils/Server.fs b/test/FsAutoComplete.Tests.Lsp/Utils/Server.fs index 57ab7a9e2..198ad2044 100644 --- a/test/FsAutoComplete.Tests.Lsp/Utils/Server.fs +++ b/test/FsAutoComplete.Tests.Lsp/Utils/Server.fs @@ -14,6 +14,7 @@ open FSharpx.Control open Expecto open Utils open Ionide.ProjInfo.Logging +open FsAutoComplete.Telemetry let private logger = LogProvider.getLoggerByName "Utils.Server" @@ -245,40 +246,44 @@ module Document = |> Observable.filter (fun n -> n.TextDocument.Uri = doc.Uri) - /// Waits (if necessary) and gets latest diagnostics. - /// - /// To detect newest diags: - /// * Waits for `fsharp/documentAnalyzed` for passed `doc` and its `doc.Version`. - /// * Then returns latest diagnostics. - /// - /// - /// ### Explanation: Get latest & correct diagnostics - /// Diagnostics aren't collected and then sent once, but instead sent after each parsing/analyzing step. - /// -> There are multiple `textDocument/publishDiagnostics` sent for each parsing/analyzing round: - /// * one when file parsed by F# compiler - /// * one for each built-in (enabled) Analyzers (in `src\FsAutoComplete\FsAutoComplete.Lsp.fs` > `FsAutoComplete.Lsp.FSharpLspServer.analyzeFile`), - /// * for linter (currently disabled) - /// * for custom analyzers - /// - /// -> To receive ALL diagnostics: use Diagnostics of last `textDocument/publishDiagnostics` event. - /// - /// Issue: What is the last `publishDiagnostics`? Might already be here or arrive in future. - /// -> `fsharp/documentAnalyzed` was introduced. Notification when a doc was completely analyzed - /// -> wait for `documentAnalyzed` - /// - /// *Inconvenience*: Only newest diags can be retrieved this way. Diags for older file versions cannot be extracted reliably: - /// `doc.Server.Events` is a `ReplaySubject` -> returns ALL previous events on new subscription - /// -> All past `documentAnalyzed` events and their diags are all received at once - /// -> waiting a bit after a version-specific `documentAnalyzed` always returns latest diags. - //ENHANCEMENT: Send `publishDiagnostics` with Doc Version (LSP `3.15.0`) -> can correlate `documentAnalyzed` and `publishDiagnostics` - let waitForLatestDiagnostics timeout (doc: Document) : Async = + let waitForLatestDiagnostics (timeout: TimeSpan) (doc: Document) : Async = async { + let tags = + seq { + "document.filepath", box doc.FilePath + "document.uri", doc.Uri + "document.version", doc.Version + } + + use _trace = OpenTelemetry.source.StartActivityForFunc(tags = tags) + logger.trace ( Log.setMessage "Waiting for diags for {uri} at version {version}" >> Log.addContext "uri" doc.Uri >> Log.addContext "version" doc.Version ) + let p: DocumentDiagnosticParams = + { WorkDoneToken = None + PartialResultToken = None + TextDocument = { Uri = doc.Uri } + Identifier = None + PreviousResultId = None } + + let! response = + Async.StartChild( + doc.Server.Server.TextDocumentDiagnostic p, + millisecondsTimeout = int timeout.TotalMilliseconds + ) + + match! response with + | Ok(DocumentDiagnosticReport.C1 d) -> return d.Items + | Ok(DocumentDiagnosticReport.C2 _) -> return Array.empty + | Result.Error e -> return failwithf "Failed to get diagnostics for %s: %A" doc.Uri e + } + + let waitForLatestPublishedDiagnostics timeout (doc: Document) : Async = + async { let mutable latest = [||] use _ = @@ -347,7 +352,6 @@ module Document = ContentChanges = [| U2.C2 { Text = text } |] } do! doc.Server.Server.TextDocumentDidChange p - do! Async.Sleep(TimeSpan.FromMilliseconds 250.) return! doc |> waitForLatestDiagnostics Helpers.defaultTimeout } @@ -359,7 +363,6 @@ module Document = // Simulate the file being written to disk so we don't hit the typechecker cache IO.File.SetLastWriteTimeUtc(doc.FilePath, DateTime.UtcNow) do! doc.Server.Server.TextDocumentDidSave p - do! Async.Sleep(TimeSpan.FromMilliseconds 250.) return! doc |> waitForLatestDiagnostics Helpers.defaultTimeout } diff --git a/test/FsAutoComplete.Tests.Lsp/Utils/Server.fsi b/test/FsAutoComplete.Tests.Lsp/Utils/Server.fsi index 66aa82cb4..ab4a28b77 100644 --- a/test/FsAutoComplete.Tests.Lsp/Utils/Server.fsi +++ b/test/FsAutoComplete.Tests.Lsp/Utils/Server.fsi @@ -106,6 +106,7 @@ module Document = /// `doc.Server.Events` is a `ReplaySubject` -> returns ALL previous events on new subscription /// -> All past `documentAnalyzed` events and their diags are all received at once /// -> waiting a bit after a version-specific `documentAnalyzed` always returns latest diags. + val waitForLatestPublishedDiagnostics: timeout: TimeSpan -> doc: Document -> Async val waitForLatestDiagnostics: timeout: TimeSpan -> doc: Document -> Async val openWith: initialText: string -> doc: Document -> Async val close: doc: Document -> Async