diff --git a/Companion.Tests/Services/SysUpgradeServiceTests.cs b/Companion.Tests/Services/SysUpgradeServiceTests.cs new file mode 100644 index 0000000..7d8a919 --- /dev/null +++ b/Companion.Tests/Services/SysUpgradeServiceTests.cs @@ -0,0 +1,55 @@ +using Companion.Services; + +namespace OpenIPC.Companion.Tests.Services; + +[TestFixture] +public class SysUpgradeServiceTests +{ + // Real /proc/mtd from an OpenIPC SSC338Q (RunCam WiFiLink) NOR device. + private const string SampleProcMtd = + "dev: size erasesize name\n" + + "mtd0: 00040000 00010000 \"boot\"\n" + + "mtd1: 00010000 00010000 \"env\"\n" + + "mtd2: 00200000 00010000 \"kernel\"\n" + + "mtd3: 00800000 00010000 \"rootfs\"\n" + + "mtd4: 005b0000 00010000 \"rootfs_data\"\n"; + + [Test] + public void ParseMtdPartitions_MapsNamesToDevicesAndSizes() + { + var partitions = SysUpgradeService.ParseMtdPartitions(SampleProcMtd); + + Assert.That(partitions["kernel"].Device, Is.EqualTo("/dev/mtd2")); + Assert.That(partitions["kernel"].SizeBytes, Is.EqualTo(0x200000)); + Assert.That(partitions["rootfs"].Device, Is.EqualTo("/dev/mtd3")); + Assert.That(partitions["rootfs"].SizeBytes, Is.EqualTo(0x800000)); + Assert.That(partitions["rootfs_data"].Device, Is.EqualTo("/dev/mtd4")); + Assert.That(partitions["rootfs_data"].SizeBytes, Is.EqualTo(0x5b0000)); + } + + [Test] + public void ParseMtdPartitions_LookupIsCaseInsensitive() + { + var partitions = SysUpgradeService.ParseMtdPartitions(SampleProcMtd); + + Assert.That(partitions.ContainsKey("ROOTFS"), Is.True); + Assert.That(partitions.TryGetValue("Kernel", out _), Is.True); + } + + [Test] + public void ParseMtdPartitions_EmptyOrGarbage_ReturnsEmpty() + { + Assert.That(SysUpgradeService.ParseMtdPartitions(""), Is.Empty); + Assert.That(SysUpgradeService.ParseMtdPartitions(null!), Is.Empty); + Assert.That(SysUpgradeService.ParseMtdPartitions("not a partition table\nrandom junk"), Is.Empty); + } + + [Test] + public void ParseMtdPartitions_HandlesCrLfAndOtherLayouts() + { + var partitions = SysUpgradeService.ParseMtdPartitions("mtd7: 00a00000 00010000 \"rootfs\"\r\n"); + + Assert.That(partitions["rootfs"].Device, Is.EqualTo("/dev/mtd7")); + Assert.That(partitions["rootfs"].SizeBytes, Is.EqualTo(0x00a00000)); + } +} diff --git a/Companion/Services/SysUpgradeService.cs b/Companion/Services/SysUpgradeService.cs index f3df80c..2eeaa54 100644 --- a/Companion/Services/SysUpgradeService.cs +++ b/Companion/Services/SysUpgradeService.cs @@ -1,6 +1,9 @@ using System; +using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Net.NetworkInformation; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Companion.Models; @@ -13,12 +16,47 @@ public class SysUpgradeService private readonly ISshClientService _sshClientService; private readonly ILogger _logger; + // Matches a /proc/mtd line, e.g. mtd3: 00800000 00010000 "rootfs" + private static readonly Regex MtdLineRegex = new( + @"^(?mtd\d+):\s+(?[0-9a-fA-F]+)\s+(?[0-9a-fA-F]+)\s+""(?[^""]+)""", + RegexOptions.Compiled); + public SysUpgradeService(ISshClientService sshClientService, ILogger logger) { _sshClientService = sshClientService; _logger = logger; } + /// A flash partition parsed from /proc/mtd. + public readonly record struct MtdPartition(string Device, long SizeBytes); + + /// + /// Parses the output of cat /proc/mtd into a name->partition map, + /// e.g. "rootfs" => { Device = "/dev/mtd3", SizeBytes = 8388608 }. + /// + public static IReadOnlyDictionary ParseMtdPartitions(string procMtdOutput) + { + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (string.IsNullOrEmpty(procMtdOutput)) + return map; + + foreach (var line in procMtdOutput.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) + { + var match = MtdLineRegex.Match(line.Trim()); + if (!match.Success) + continue; + + if (!long.TryParse(match.Groups["size"].Value, NumberStyles.HexNumber, + CultureInfo.InvariantCulture, out var size)) + continue; + + var name = match.Groups["name"].Value.Trim(); + map[name] = new MtdPartition($"/dev/{match.Groups["dev"].Value}", size); + } + + return map; + } + public async Task PerformSysupgradeAsync(DeviceConfig deviceConfig, string kernelPath, string rootfsPath, Action updateProgress, CancellationToken cancellationToken) { @@ -34,19 +72,37 @@ public async Task PerformSysupgradeAsync(DeviceConfig deviceConfig, string kerne await UploadAndVerifyWithRetryAsync(deviceConfig, rootfsPath, remoteRootfsPath, "rootfs", updateProgress, cancellationToken); updateProgress("Root filesystem binary uploaded successfully."); - updateProgress("Starting sysupgrade. Do not unplug the device."); - await _sshClientService.ExecuteCommandWithProgressAsync( - deviceConfig, - $"sysupgrade --force_ver -n -z --kernel={OpenIPC.RemoteTempFolder}/{kernelFilename} --rootfs={OpenIPC.RemoteTempFolder}/{rootfsFilename}", - updateProgress, - cancellationToken, - timeout: TimeSpan.FromMinutes(15), - allowDisconnectCompletion: true, - disableTimeout: true - ); + // sysupgrade loop-mounts the new rootfs to verify it before writing. On a device whose + // running kernel lacks the squashfs compressor of the new image (commonly XZ), that mount + // fails with "mount: ... Invalid argument" and sysupgrade aborts *after* it has already + // flashed the kernel, leaving a half-upgraded unit. Probe the exact same mount first; if the + // running kernel can't mount the image, skip sysupgrade and write the partitions directly + // with flashcp (a raw write needs no mount) — the only thing that works on those units. + if (await CanRunningKernelMountRootfsAsync(deviceConfig, remoteRootfsPath, cancellationToken)) + { + updateProgress("Starting sysupgrade. Do not unplug the device."); + await _sshClientService.ExecuteCommandWithProgressAsync( + deviceConfig, + $"sysupgrade --force_ver -n -z --kernel={OpenIPC.RemoteTempFolder}/{kernelFilename} --rootfs={OpenIPC.RemoteTempFolder}/{rootfsFilename}", + updateProgress, + cancellationToken, + timeout: TimeSpan.FromMinutes(15), + allowDisconnectCompletion: true, + disableTimeout: true + ); + } + else + { + updateProgress( + "This device's running kernel cannot mount the new root filesystem, so 'sysupgrade' " + + "would abort during verification. Flashing the partitions directly instead."); + await FlashImageDirectlyAsync( + deviceConfig, kernelPath, rootfsPath, remoteKernelPath, remoteRootfsPath, + updateProgress, cancellationToken); + } await WaitForDeviceRecoveryAsync(deviceConfig, updateProgress, cancellationToken); - updateProgress("Sysupgrade process completed and device reconnected."); + updateProgress("Firmware update completed and device reconnected."); } catch (Exception ex) { @@ -97,6 +153,224 @@ private async Task UploadAndVerifyWithRetryAsync( } } + /// + /// Seconds the device is given to complete the verify-mount before we stop waiting on it. + /// + private const int MountProbeSeconds = 45; + + /// + /// Returns true if the device's currently-running kernel can loop-mount the uploaded rootfs + /// squashfs. This is exactly what sysupgrade does to verify the image before flashing, so + /// it predicts whether sysupgrade will succeed or abort with "mount ... Invalid argument". + /// + /// + /// The mount does not always fail cleanly — it can block indefinitely. Observed on an + /// SSC338Q air unit: sysupgrade printed "Update rootfs from /tmp/rootfs.squashfs.ssc338q" and + /// never emitted another byte, because everything between that line and the flash write is a + /// silent losetup+mount. An unbounded probe would therefore hang in exactly the case this + /// fallback exists to survive, so it is bounded twice, and a probe that does not answer in time + /// counts as NOT mountable — handing such a device to sysupgrade would only wedge it on the very + /// same mount. + /// + private async Task CanRunningKernelMountRootfsAsync( + DeviceConfig deviceConfig, + string remoteRootfsPath, + CancellationToken cancellationToken) + { + // Mount read-only via loop, print a sentinel only on success, then always clean up. + // `timeout` keeps the device-side mount from lingering forever; it is best-effort only, + // since the SIGTERM it sends cannot free a mount wedged in uninterruptible (D) state, and + // the applet may be absent on a minimal build. Hence the client-side bound below as well. + // + // The bare, un-timed mount is used ONLY when the `timeout` applet is missing — never as a + // fallback after a mount failure/timeout. `M` holds `timeout N` when the applet exists and is + // empty otherwise, so exactly one mount runs: a bounded one where possible. A `... || mount` + // form would spawn a second unbounded mount on every real mount failure, which — because the + // client-side wait below abandons the command without killing it — is precisely the + // background-wedge this probe exists to avoid. + const string sentinel = "RUBY_MOUNT_OK"; + var probe = + $"d=$(mktemp -d 2>/dev/null || echo /tmp/.cmp_verify); mkdir -p \"$d\"; " + + $"if command -v timeout >/dev/null 2>&1; then M=\"timeout {MountProbeSeconds}\"; else M=\"\"; fi; " + + $"if $M mount -t squashfs -o loop,ro '{remoteRootfsPath}' \"$d\" 2>/dev/null; then " + + $"echo {sentinel}; umount \"$d\" 2>/dev/null; fi; rmdir \"$d\" 2>/dev/null; true"; + + try + { + // A CancellationToken cannot rescue us here: ExecuteCommandWithResponseAsync runs the + // blocking SSH.NET RunCommand inside Task.Run, whose token only prevents the delegate + // from *starting* — once it is running, cancelling it does nothing and the await would + // wait forever. Bound it on the wall clock instead. + var probeTask = _sshClientService.ExecuteCommandWithResponseAsync(deviceConfig, probe, cancellationToken); + var limit = Task.Delay(TimeSpan.FromSeconds(MountProbeSeconds + 20), cancellationToken); + + if (await Task.WhenAny(probeTask, limit) != probeTask) + { + cancellationToken.ThrowIfCancellationRequested(); + _logger.Warning( + "Rootfs mount probe did not answer within {Seconds}s — the mount is wedged. " + + "Treating rootfs as NOT mountable (using direct flashcp).", + MountProbeSeconds + 20); + return false; + } + + var result = await probeTask; + var ok = result?.Result?.Contains(sentinel, StringComparison.Ordinal) == true; + _logger.Information("Rootfs mount probe: {Result}.", + ok ? "mountable (using sysupgrade)" : "NOT mountable (using direct flashcp)"); + return ok; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + // If the probe itself can't run, fall back to the existing behaviour (try sysupgrade). + _logger.Warning(ex, "Rootfs mount probe failed to execute; assuming sysupgrade is usable."); + return true; + } + } + + /// + /// Writes the kernel and rootfs straight to their MTD partitions with flashcp (no mount needed), + /// erases the settings overlay, and reboots — mirroring what sysupgrade --force_ver -n + /// would have done, for devices where sysupgrade's verify-mount fails. Partitions are looked up by + /// name from /proc/mtd and size-checked, so we never write the wrong or an oversized partition. + /// + private async Task FlashImageDirectlyAsync( + DeviceConfig deviceConfig, + string kernelPath, + string rootfsPath, + string remoteKernelPath, + string remoteRootfsPath, + Action updateProgress, + CancellationToken cancellationToken) + { + updateProgress("Reading device partition table..."); + var mtdResult = await _sshClientService.ExecuteCommandWithResponseAsync(deviceConfig, "cat /proc/mtd", cancellationToken); + var partitions = ParseMtdPartitions(mtdResult?.Result ?? string.Empty); + + if (!partitions.TryGetValue("kernel", out var kernelPartition) || + !partitions.TryGetValue("rootfs", out var rootfsPartition)) + throw new InvalidOperationException( + "Could not find 'kernel' and 'rootfs' partitions in /proc/mtd; aborting direct flash to avoid writing the wrong partition."); + + // Refuse to write an image larger than its partition (would corrupt the adjacent partition). + var kernelSize = new FileInfo(kernelPath).Length; + var rootfsSize = new FileInfo(rootfsPath).Length; + if (kernelSize > kernelPartition.SizeBytes) + throw new InvalidOperationException( + $"Kernel ({kernelSize} bytes) is larger than its flash partition {kernelPartition.Device} ({kernelPartition.SizeBytes} bytes). Aborting."); + if (rootfsSize > rootfsPartition.SizeBytes) + throw new InvalidOperationException( + $"Root filesystem ({rootfsSize} bytes) is larger than its flash partition {rootfsPartition.Device} ({rootfsPartition.SizeBytes} bytes). Aborting."); + + if (!await RemoteCommandExistsAsync(deviceConfig, "flashcp", cancellationToken)) + throw new InvalidOperationException("'flashcp' (mtd-utils) is not available on the device; cannot flash directly."); + + updateProgress($"Flashing kernel to {kernelPartition.Device}. Do not unplug the device."); + await FlashPartitionAsync( + deviceConfig, remoteKernelPath, kernelPartition.Device, "kernel", + TimeSpan.FromMinutes(5), updateProgress, cancellationToken); + + updateProgress($"Flashing root filesystem to {rootfsPartition.Device}. Do not unplug the device."); + await FlashPartitionAsync( + deviceConfig, remoteRootfsPath, rootfsPartition.Device, "root filesystem", + TimeSpan.FromMinutes(15), updateProgress, cancellationToken); + + // sysupgrade -n resets the settings overlay; replicate that so stale config from the old + // firmware doesn't shadow the new image. Best-effort: skip if there is no such partition. + if (partitions.TryGetValue("rootfs_data", out var overlayPartition)) + { + updateProgress($"Erasing settings overlay {overlayPartition.Device}..."); + await _sshClientService.ExecuteCommandAsync(deviceConfig, $"flash_eraseall {overlayPartition.Device}"); + } + + updateProgress("Flash complete. Rebooting device. Do not unplug the device."); + await _sshClientService.ExecuteCommandWithProgressAsync( + deviceConfig, + "reboot", + updateProgress, + cancellationToken, + timeout: TimeSpan.FromMinutes(2), + allowDisconnectCompletion: true, + disableTimeout: true); + } + + /// + /// Writes one image to one MTD partition with flashcp -v, blocking until the write reports + /// an explicit exit code, and throwing if it failed or never completed. + /// + /// + /// decides a command is finished + /// only when it prints a sysupgrade-style sentinel or the SSH session drops. flashcp does neither — + /// it streams progress and returns to the shell prompt — so with disableTimeout the read loop + /// would spin forever, and even on timeout the helper only reports via progress text without + /// throwing. Raw MTD writes must fail loudly, so we append ; echo MARKER$? to carry flashcp's + /// exit status back over the shell stream, complete the moment that marker is parsed, keep a finite + /// timeout, and throw on a non-zero code or on no marker at all (timeout / lost connection). + /// + private async Task FlashPartitionAsync( + DeviceConfig deviceConfig, + string remoteImagePath, + string partitionDevice, + string label, + TimeSpan timeout, + Action updateProgress, + CancellationToken cancellationToken) + { + const string marker = "RUBY_FLASHCP_EXIT:"; + int? exitCode = null; + + // Parse the "MARKER" line flashcp's shell prints on exit. The shell also echoes the + // command itself, which contains the literal "MARKER$?" (no digits) — skipping the no-digit + // case keeps that echo from being mistaken for completion. + void Sniff(string line) + { + updateProgress(line); + var idx = line.IndexOf(marker, StringComparison.Ordinal); + if (idx < 0) + return; + var end = idx + marker.Length; + while (end < line.Length && char.IsWhiteSpace(line[end])) + end++; + var start = end; + while (end < line.Length && char.IsDigit(line[end])) + end++; + if (end > start && int.TryParse(line.Substring(start, end - start), out var code)) + exitCode = code; + } + + await _sshClientService.ExecuteCommandWithProgressAsync( + deviceConfig, + $"flashcp -v '{remoteImagePath}' {partitionDevice}; echo {marker}$?", + Sniff, + cancellationToken, + timeout: timeout, + // Sniff runs before this check each line, so "we parsed a real exit code" is completion. + isCommandComplete: _ => exitCode.HasValue, + disableTimeout: false); + + if (exitCode is null) + throw new InvalidOperationException( + $"Flashing {label} to {partitionDevice} did not report completion within {timeout.TotalMinutes:0} min " + + "(flashcp may be wedged or the connection dropped); aborting to avoid a silently half-written flash."); + if (exitCode != 0) + throw new InvalidOperationException( + $"flashcp failed writing {label} to {partitionDevice} (exit {exitCode}); the flash is likely incomplete."); + } + + private async Task RemoteCommandExistsAsync( + DeviceConfig deviceConfig, + string command, + CancellationToken cancellationToken) + { + var result = await _sshClientService.ExecuteCommandWithResponseAsync( + deviceConfig, $"command -v {command} >/dev/null 2>&1 && echo FOUND", cancellationToken); + return result?.Result?.Contains("FOUND", StringComparison.Ordinal) == true; + } + private async Task ValidateRemoteFileSizeAsync( DeviceConfig deviceConfig, string localPath,