-
-
Notifications
You must be signed in to change notification settings - Fork 230
Expand file tree
/
Copy pathMemoryMonitor.cs
More file actions
225 lines (190 loc) · 7.36 KB
/
MemoryMonitor.cs
File metadata and controls
225 lines (190 loc) · 7.36 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
/*
* dotnet-gcdump needs .NET 6 or later:
* https://www.nuget.org/packages/dotnet-gcdump#supportedframeworks-body-tab
*
* Also `GC.GetGCMemoryInfo()` is not available in NetFX or NetStandard
*/
#if MEMORY_DUMP_SUPPORTED
using Sentry.Extensibility;
using Sentry.Internal.Extensions;
namespace Sentry.Internal;
internal sealed class MemoryMonitor : IDisposable
{
private readonly long _totalMemory;
private readonly SentryOptions _options;
private readonly HeapDumpOptions _dumpOptions;
private readonly CancellationTokenSource _cancellationTokenSource = new();
private readonly Action _onCaptureDump; // Just for testing purposes
private readonly Action<string> _onDumpCollected;
private Task? _monitorTask;
public MemoryMonitor(SentryOptions options, Action<string> onDumpCollected, Action? onCaptureDump = null, IGCImplementation? gc = null)
{
if (options.HeapDumpOptions is null)
{
throw new ArgumentException("No heap dump options provided", nameof(options));
}
_options = options;
_dumpOptions = options.HeapDumpOptions;
_onDumpCollected = onDumpCollected;
_onCaptureDump = onCaptureDump ?? CaptureMemoryDump;
gc ??= new SystemGCImplementation();
_totalMemory = gc.TotalAvailableMemoryBytes;
// Since we're not awaiting the task, the continuation will happen elsewhere but that's OK - all we care about
// is that any exceptions get logged as soon as possible.
_monitorTask = GarbageCollectionMonitor.Start(CheckMemoryUsage, _cancellationTokenSource.Token, gc)
.ContinueWith(
t => _options.LogError(t.Exception!, "Garbage collection monitor failed"),
TaskContinuationOptions.OnlyOnFaulted // guarantees that the exception is not null
);
}
internal void CheckMemoryUsage()
{
var eventTime = DateTimeOffset.UtcNow;
if (!_dumpOptions.Debouncer.CanProcess(eventTime))
{
return;
}
var usedMemory = Environment.WorkingSet;
if (!_dumpOptions.Trigger(usedMemory, _totalMemory))
{
return;
}
_dumpOptions.Debouncer.RecordOccurence(eventTime);
var usedMemoryPercentage = ((double)usedMemory / _totalMemory) * 100;
_options.LogDebug("Auto heap dump triggered: Total: {0:N0} bytes, Used: {1:N0} bytes ({2:N2}%)",
_totalMemory, usedMemory, usedMemoryPercentage);
_onCaptureDump();
}
public void CaptureMemoryDump() => CaptureMemoryDump(DefaultProcessRunner);
/// <summary>
/// Override used for testing
/// </summary>
internal void CaptureMemoryDump(Action<string> dumpProcessRunner)
{
if (_options.DisableFileWrite)
{
_options.LogDebug("File write has been disabled via the options. Unable to create memory dump.");
return;
}
var dumpFile = TryGetDumpLocation();
if (dumpFile is null)
{
return;
}
var command = $"dotnet-gcdump collect -v -p {Environment.ProcessId} -o '{dumpFile}'";
dumpProcessRunner.Invoke(command);
if (!_options.FileSystem.FileExists(dumpFile))
{
// if this happens, hopefully there would be more information in the standard output from the process above
_options.LogError("Unexpected error creating memory dump. Use debug-level to see output of dotnet-gcdump.");
return;
}
const long maxDumpSizeInBytes = 20 * 1024 * 1024;
var fileInfo = new FileInfo(dumpFile);
if (fileInfo.Length > maxDumpSizeInBytes)
{
_options.LogWarning($"Memory dump file is too large ({fileInfo.Length} bytes). and will not be attached to the Sentry event.");
try
{
File.Delete(dumpFile);
}
catch (Exception ex)
{
_options.LogError($"Error deleting memory dump file: {ex.Message}");
}
return;
}
_onDumpCollected(dumpFile);
try
{
File.Delete(dumpFile);
}
catch (Exception ex)
{
_options.LogError($"Error deleting memory dump file after sending to Sentry: {ex.Message}");
}
}
/// <summary>
/// Default process runner for the memory dump command
/// </summary>
/// <remarks>
/// This code hangs the test runners on Windows in CI so must be mocked for testing.
/// </remarks>
private void DefaultProcessRunner(string command)
{
_options.LogDebug($"Starting process: {command}");
using var process = new Process();
process.StartInfo = new ProcessStartInfo
{
FileName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "cmd.exe" : "/bin/bash",
Arguments = $"-c \"{command}\"",
RedirectStandardOutput = true,
RedirectStandardError = false,
UseShellExecute = false,
CreateNoWindow = true,
};
process.Start();
while (!process.StandardOutput.EndOfStream)
{
if (process.StandardOutput.ReadLine() is { } line)
{
_options.LogDebug($"gcdump: {line}");
}
}
#if NET8_0_OR_GREATER
process.WaitForExit(TimeSpan.FromSeconds(5));
#else
process.WaitForExit(5000);
#endif
}
internal string? TryGetDumpLocation()
{
try
{
var rootPath = _options.CacheDirectoryPath ??
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
var directoryPath = Path.Combine(rootPath, "Sentry", _options.Dsn!.GetHashString());
var fileSystem = _options.FileSystem;
if (!fileSystem.CreateDirectory(directoryPath))
{
_options.LogWarning("Failed to create a directory for memory dump ({0}).", directoryPath);
return null;
}
_options.LogDebug("Created directory for heap dump ({0}).", directoryPath);
var timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
var processId = Environment.ProcessId;
var filePath = Path.Combine(directoryPath, $"{timestamp}_{processId}.gcdump");
if (fileSystem.FileExists(filePath))
{
_options.LogWarning("Duplicate dump file detected.");
return null;
}
return filePath;
}
// If there's no write permission or the platform doesn't support this, we handle simply log and bug out
catch (Exception ex)
{
_options.LogError(ex, "Failed to resolve appropriate memory dump location.");
return null;
}
}
public void Dispose()
{
// Important no exceptions can be thrown from this method as it's called when disposing the Hub
_cancellationTokenSource.Cancel();
try
{
_monitorTask?.Wait(500); // This should complete very quickly (possibly before we even wait)
}
catch (OperationCanceledException)
{
// Ignore
}
catch (Exception e)
{
_options.LogError(e, "Error waiting for GarbageCollectionMonitor task to complete");
}
_cancellationTokenSource.Dispose();
}
}
#endif