Flash directly via flashcp when sysupgrade's verify-mount fails

Companion.Tests/Services/SysUpgradeServiceTests.cs

@@ -0,0 +1,55 @@
+using Companion.Services;
+
+namespace OpenIPC.Companion.Tests.Services;
+
+[TestFixture]
+public class SysUpgradeServiceTests
+{
+    // Real /proc/mtd from an OpenIPC SSC338Q (RunCam WiFiLink) NOR device.
+    private const string SampleProcMtd =
+        "dev:    size   erasesize  name\n" +
+        "mtd0: 00040000 00010000 \"boot\"\n" +
+        "mtd1: 00010000 00010000 \"env\"\n" +
+        "mtd2: 00200000 00010000 \"kernel\"\n" +
+        "mtd3: 00800000 00010000 \"rootfs\"\n" +
+        "mtd4: 005b0000 00010000 \"rootfs_data\"\n";
+
+    [Test]
+    public void ParseMtdPartitions_MapsNamesToDevicesAndSizes()
+    {
+        var partitions = SysUpgradeService.ParseMtdPartitions(SampleProcMtd);
+
+        Assert.That(partitions["kernel"].Device, Is.EqualTo("/dev/mtd2"));
+        Assert.That(partitions["kernel"].SizeBytes, Is.EqualTo(0x200000));
+        Assert.That(partitions["rootfs"].Device, Is.EqualTo("/dev/mtd3"));
+        Assert.That(partitions["rootfs"].SizeBytes, Is.EqualTo(0x800000));
+        Assert.That(partitions["rootfs_data"].Device, Is.EqualTo("/dev/mtd4"));
+        Assert.That(partitions["rootfs_data"].SizeBytes, Is.EqualTo(0x5b0000));
+    }
+
+    [Test]
+    public void ParseMtdPartitions_LookupIsCaseInsensitive()
+    {
+        var partitions = SysUpgradeService.ParseMtdPartitions(SampleProcMtd);
+
+        Assert.That(partitions.ContainsKey("ROOTFS"), Is.True);
+        Assert.That(partitions.TryGetValue("Kernel", out _), Is.True);
+    }
+
+    [Test]
+    public void ParseMtdPartitions_EmptyOrGarbage_ReturnsEmpty()
+    {
+        Assert.That(SysUpgradeService.ParseMtdPartitions(""), Is.Empty);
+        Assert.That(SysUpgradeService.ParseMtdPartitions(null!), Is.Empty);
+        Assert.That(SysUpgradeService.ParseMtdPartitions("not a partition table\nrandom junk"), Is.Empty);
+    }
+
+    [Test]
+    public void ParseMtdPartitions_HandlesCrLfAndOtherLayouts()
+    {
+        var partitions = SysUpgradeService.ParseMtdPartitions("mtd7: 00a00000 00010000 \"rootfs\"\r\n");
+
+        Assert.That(partitions["rootfs"].Device, Is.EqualTo("/dev/mtd7"));
+        Assert.That(partitions["rootfs"].SizeBytes, Is.EqualTo(0x00a00000));
+    }
+}

Companion/Services/SysUpgradeService.cs

@@ -1,6 +1,9 @@
 using System;
+using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Net.NetworkInformation;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using Companion.Models;
@@ -13,12 +16,47 @@ public class SysUpgradeService
     private readonly ISshClientService _sshClientService;
     private readonly ILogger _logger;
 
+    // Matches a /proc/mtd line, e.g.   mtd3: 00800000 00010000 "rootfs"
+    private static readonly Regex MtdLineRegex = new(
+        @"^(?<dev>mtd\d+):\s+(?<size>[0-9a-fA-F]+)\s+(?<erasesize>[0-9a-fA-F]+)\s+""(?<name>[^""]+)""",
+        RegexOptions.Compiled);
+
     public SysUpgradeService(ISshClientService sshClientService, ILogger logger)
     {
         _sshClientService = sshClientService;
         _logger = logger;
     }
 
+    /// <summary>A flash partition parsed from <c>/proc/mtd</c>.</summary>
+    public readonly record struct MtdPartition(string Device, long SizeBytes);
+
+    /// <summary>
+    /// Parses the output of <c>cat /proc/mtd</c> into a name-&gt;partition map,
+    /// e.g. "rootfs" =&gt; { Device = "/dev/mtd3", SizeBytes = 8388608 }.
+    /// </summary>
+    public static IReadOnlyDictionary<string, MtdPartition> ParseMtdPartitions(string procMtdOutput)
+    {
+        var map = new Dictionary<string, MtdPartition>(StringComparer.OrdinalIgnoreCase);
+        if (string.IsNullOrEmpty(procMtdOutput))
+            return map;
+
+        foreach (var line in procMtdOutput.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries))
+        {
+            var match = MtdLineRegex.Match(line.Trim());
+            if (!match.Success)
+                continue;
+
+            if (!long.TryParse(match.Groups["size"].Value, NumberStyles.HexNumber,
+                    CultureInfo.InvariantCulture, out var size))
+                continue;
+
+            var name = match.Groups["name"].Value.Trim();
+            map[name] = new MtdPartition($"/dev/{match.Groups["dev"].Value}", size);
+        }
+
+        return map;
+    }
+
     public async Task PerformSysupgradeAsync(DeviceConfig deviceConfig, string kernelPath, string rootfsPath, 
         Action<string> updateProgress, CancellationToken cancellationToken)
     {
@@ -34,19 +72,37 @@ public async Task PerformSysupgradeAsync(DeviceConfig deviceConfig, string kerne
             await UploadAndVerifyWithRetryAsync(deviceConfig, rootfsPath, remoteRootfsPath, "rootfs", updateProgress, cancellationToken);
             updateProgress("Root filesystem binary uploaded successfully.");
 
-            updateProgress("Starting sysupgrade. Do not unplug the device.");
-            await _sshClientService.ExecuteCommandWithProgressAsync(
-                deviceConfig,
-                $"sysupgrade --force_ver -n -z --kernel={OpenIPC.RemoteTempFolder}/{kernelFilename} --rootfs={OpenIPC.RemoteTempFolder}/{rootfsFilename}",
-                updateProgress,
-                cancellationToken,
-                timeout: TimeSpan.FromMinutes(15),
-                allowDisconnectCompletion: true,
-                disableTimeout: true
-            );
+            // sysupgrade loop-mounts the new rootfs to verify it before writing. On a device whose
+            // running kernel lacks the squashfs compressor of the new image (commonly XZ), that mount
+            // fails with "mount: ... Invalid argument" and sysupgrade aborts *after* it has already
+            // flashed the kernel, leaving a half-upgraded unit. Probe the exact same mount first; if the
+            // running kernel can't mount the image, skip sysupgrade and write the partitions directly
+            // with flashcp (a raw write needs no mount) — the only thing that works on those units.
+            if (await CanRunningKernelMountRootfsAsync(deviceConfig, remoteRootfsPath, cancellationToken))
+            {
+                updateProgress("Starting sysupgrade. Do not unplug the device.");
+                await _sshClientService.ExecuteCommandWithProgressAsync(
+                    deviceConfig,
+                    $"sysupgrade --force_ver -n -z --kernel={OpenIPC.RemoteTempFolder}/{kernelFilename} --rootfs={OpenIPC.RemoteTempFolder}/{rootfsFilename}",
+                    updateProgress,
+                    cancellationToken,
+                    timeout: TimeSpan.FromMinutes(15),
+                    allowDisconnectCompletion: true,
+                    disableTimeout: true
+                );
+            }
+            else
+            {
+                updateProgress(
+                    "This device's running kernel cannot mount the new root filesystem, so 'sysupgrade' " +
+                    "would abort during verification. Flashing the partitions directly instead.");
+                await FlashImageDirectlyAsync(
+                    deviceConfig, kernelPath, rootfsPath, remoteKernelPath, remoteRootfsPath,
+                    updateProgress, cancellationToken);
+            }
 
             await WaitForDeviceRecoveryAsync(deviceConfig, updateProgress, cancellationToken);
-            updateProgress("Sysupgrade process completed and device reconnected.");
+            updateProgress("Firmware update completed and device reconnected.");
         }
         catch (Exception ex)
         {
@@ -97,6 +153,162 @@ private async Task UploadAndVerifyWithRetryAsync(
         }
     }
 
+    /// <summary>
+    /// Seconds the device is given to complete the verify-mount before we stop waiting on it.
+    /// </summary>
+    private const int MountProbeSeconds = 45;
+
+    /// <summary>
+    /// Returns true if the device's currently-running kernel can loop-mount the uploaded rootfs
+    /// squashfs. This is exactly what <c>sysupgrade</c> does to verify the image before flashing, so
+    /// it predicts whether sysupgrade will succeed or abort with "mount ... Invalid argument".
+    /// </summary>
+    /// <remarks>
+    /// The mount does not always fail cleanly — it can <b>block indefinitely</b>. Observed on an
+    /// SSC338Q air unit: sysupgrade printed "Update rootfs from /tmp/rootfs.squashfs.ssc338q" and
+    /// never emitted another byte, because everything between that line and the flash write is a
+    /// silent losetup+mount. An unbounded probe would therefore hang in exactly the case this
+    /// fallback exists to survive, so it is bounded twice, and a probe that does not answer in time
+    /// counts as NOT mountable — handing such a device to sysupgrade would only wedge it on the very
+    /// same mount.
+    /// </remarks>
+    private async Task<bool> CanRunningKernelMountRootfsAsync(
+        DeviceConfig deviceConfig,
+        string remoteRootfsPath,
+        CancellationToken cancellationToken)
+    {
+        // Mount read-only via loop, print a sentinel only on success, then always clean up.
+        // `timeout` keeps the device-side mount from lingering forever; it is best-effort only,
+        // since the SIGTERM it sends cannot free a mount wedged in uninterruptible (D) state, and
+        // the applet may be absent on a minimal build. Hence the client-side bound below as well.
+        const string sentinel = "RUBY_MOUNT_OK";
+        var probe =
+            $"d=$(mktemp -d 2>/dev/null || echo /tmp/.cmp_verify); mkdir -p \"$d\"; " +
+            $"if timeout {MountProbeSeconds} mount -t squashfs -o loop,ro '{remoteRootfsPath}' \"$d\" 2>/dev/null " +
+            $"|| mount -t squashfs -o loop,ro '{remoteRootfsPath}' \"$d\" 2>/dev/null; then " +
+            $"echo {sentinel}; umount \"$d\" 2>/dev/null; fi; rmdir \"$d\" 2>/dev/null; true";
+
+        try
+        {
+            // A CancellationToken cannot rescue us here: ExecuteCommandWithResponseAsync runs the
+            // blocking SSH.NET RunCommand inside Task.Run, whose token only prevents the delegate
+            // from *starting* — once it is running, cancelling it does nothing and the await would
+            // wait forever. Bound it on the wall clock instead.
+            var probeTask = _sshClientService.ExecuteCommandWithResponseAsync(deviceConfig, probe, cancellationToken);
+            var limit = Task.Delay(TimeSpan.FromSeconds(MountProbeSeconds + 20), cancellationToken);
+
+            if (await Task.WhenAny(probeTask, limit) != probeTask)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+                _logger.Warning(
+                    "Rootfs mount probe did not answer within {Seconds}s — the mount is wedged. " +
+                    "Treating rootfs as NOT mountable (using direct flashcp).",
+                    MountProbeSeconds + 20);
+                return false;
+            }
+
+            var result = await probeTask;
+            var ok = result?.Result?.Contains(sentinel, StringComparison.Ordinal) == true;
+            _logger.Information("Rootfs mount probe: {Result}.",
+                ok ? "mountable (using sysupgrade)" : "NOT mountable (using direct flashcp)");
+            return ok;
+        }
+        catch (OperationCanceledException)
+        {
+            throw;
+        }
+        catch (Exception ex)
+        {
+            // If the probe itself can't run, fall back to the existing behaviour (try sysupgrade).
+            _logger.Warning(ex, "Rootfs mount probe failed to execute; assuming sysupgrade is usable.");
+            return true;
+        }
+    }
+
+    /// <summary>
+    /// Writes the kernel and rootfs straight to their MTD partitions with flashcp (no mount needed),
+    /// erases the settings overlay, and reboots — mirroring what <c>sysupgrade --force_ver -n</c>
+    /// would have done, for devices where sysupgrade's verify-mount fails. Partitions are looked up by
+    /// name from /proc/mtd and size-checked, so we never write the wrong or an oversized partition.
+    /// </summary>
+    private async Task FlashImageDirectlyAsync(
+        DeviceConfig deviceConfig,
+        string kernelPath,
+        string rootfsPath,
+        string remoteKernelPath,
+        string remoteRootfsPath,
+        Action<string> updateProgress,
+        CancellationToken cancellationToken)
+    {
+        updateProgress("Reading device partition table...");
+        var mtdResult = await _sshClientService.ExecuteCommandWithResponseAsync(deviceConfig, "cat /proc/mtd", cancellationToken);
+        var partitions = ParseMtdPartitions(mtdResult?.Result ?? string.Empty);
+
+        if (!partitions.TryGetValue("kernel", out var kernelPartition) ||
+            !partitions.TryGetValue("rootfs", out var rootfsPartition))
+            throw new InvalidOperationException(
+                "Could not find 'kernel' and 'rootfs' partitions in /proc/mtd; aborting direct flash to avoid writing the wrong partition.");
+
+        // Refuse to write an image larger than its partition (would corrupt the adjacent partition).
+        var kernelSize = new FileInfo(kernelPath).Length;
+        var rootfsSize = new FileInfo(rootfsPath).Length;
+        if (kernelSize > kernelPartition.SizeBytes)
+            throw new InvalidOperationException(
+                $"Kernel ({kernelSize} bytes) is larger than its flash partition {kernelPartition.Device} ({kernelPartition.SizeBytes} bytes). Aborting.");
+        if (rootfsSize > rootfsPartition.SizeBytes)
+            throw new InvalidOperationException(
+                $"Root filesystem ({rootfsSize} bytes) is larger than its flash partition {rootfsPartition.Device} ({rootfsPartition.SizeBytes} bytes). Aborting.");
+
+        if (!await RemoteCommandExistsAsync(deviceConfig, "flashcp", cancellationToken))
+            throw new InvalidOperationException("'flashcp' (mtd-utils) is not available on the device; cannot flash directly.");
+
+        updateProgress($"Flashing kernel to {kernelPartition.Device}. Do not unplug the device.");
+        await _sshClientService.ExecuteCommandWithProgressAsync(
+            deviceConfig,
+            $"flashcp -v '{remoteKernelPath}' {kernelPartition.Device}",
+            updateProgress,
+            cancellationToken,
+            timeout: TimeSpan.FromMinutes(5),
+            disableTimeout: true);
+
+        updateProgress($"Flashing root filesystem to {rootfsPartition.Device}. Do not unplug the device.");
+        await _sshClientService.ExecuteCommandWithProgressAsync(
+            deviceConfig,
+            $"flashcp -v '{remoteRootfsPath}' {rootfsPartition.Device}",
+            updateProgress,
+            cancellationToken,
+            timeout: TimeSpan.FromMinutes(15),
+            disableTimeout: true);
+
+        // sysupgrade -n resets the settings overlay; replicate that so stale config from the old
+        // firmware doesn't shadow the new image. Best-effort: skip if there is no such partition.
+        if (partitions.TryGetValue("rootfs_data", out var overlayPartition))
+        {
+            updateProgress($"Erasing settings overlay {overlayPartition.Device}...");
+            await _sshClientService.ExecuteCommandAsync(deviceConfig, $"flash_eraseall {overlayPartition.Device}");
+        }
+
+        updateProgress("Flash complete. Rebooting device. Do not unplug the device.");
+        await _sshClientService.ExecuteCommandWithProgressAsync(
+            deviceConfig,
+            "reboot",
+            updateProgress,
+            cancellationToken,
+            timeout: TimeSpan.FromMinutes(2),
+            allowDisconnectCompletion: true,
+            disableTimeout: true);
+    }
+
+    private async Task<bool> RemoteCommandExistsAsync(
+        DeviceConfig deviceConfig,
+        string command,
+        CancellationToken cancellationToken)
+    {
+        var result = await _sshClientService.ExecuteCommandWithResponseAsync(
+            deviceConfig, $"command -v {command} >/dev/null 2>&1 && echo FOUND", cancellationToken);
+        return result?.Result?.Contains("FOUND", StringComparison.Ordinal) == true;
+    }
+
     private async Task ValidateRemoteFileSizeAsync(
         DeviceConfig deviceConfig,
         string localPath,