feat(flash-backup): streaming download + save-to-server, with compression & progress

emhttp/languages/en_US/helptext.txt

@@ -443,6 +443,8 @@ array or the cache disk/pool.
 
 :flash_backup_help:
 Use *Boot device backup* to create a single zip file of the current contents of the boot device and store it locally on your computer.
+
+The backup runs in the background and reports its progress, then downloads automatically when finished. Choose a *Compression level* before starting: *None* is the fastest but produces the largest file, while *Maximum* is the slowest but produces the smallest file.
 :end
 
 :syslinux_cfg_help:

emhttp/plugins/dynamix/BootInfo.page

@@ -16,36 +16,53 @@ Tag="usb"
 ?>
 <?$tpmEnabled = !empty($var['flashGUID']) && strpos($var['flashGUID'], '01-') !== 0 && !empty($var['tpmGUID']) ? true : false;?>
 <script>
-function cleanUp(zip) {
+// Run the flash backup as a tracked background job through StartCommand.php
+// (same mechanism as plugin/docker operations) and stream progress into the
+// standard nchan progress window. The script publishes a _FILE_<name> message
+// so we know which archive to download once it reports _DONE_.
+var backupFile = "";
+var nchan_backup = new NchanSubscriber('/sub/flash_backup',{subscriber:'websocket', reconnectTimeout:5000});
+
+nchan_backup.on('message', function(data) {
+  if (!data) return;
+  if (data.startsWith('_FILE_')) { backupFile = data.substring(6); return; }
+  if (openError(data)) { nchan_backup.stop(); return; }
+  if (openDone(data)) {
+    nchan_backup.stop();
+    if (backupFile) {
+      // archive ready: trigger the browser download, then clean it up afterwards
+      location = '/'+backupFile;
+      setTimeout(cleanUpBackup,6000);
+    }
+    return;
+  }
+  var box = $('pre#swaltext');
+  box.html(box.html()+'<br>'+data).scrollTop(box[0].scrollHeight);
+});
+
+function cleanUpBackup() {
   if (document.hasFocus()) {
-    $('input[value="_(Creating boot device backup)_..."]').val("_(Boot device backup)_").prop('disabled',false);
-    $('div.spinner').hide('slow');
-    $('#pleaseWait').hide('slow');
-    $.post('/webGui/include/Download.php',{cmd:'unlink',file:zip});
+    $.post('/webGui/include/Download.php',{cmd:'unlink',file:backupFile});
+    backupFile = "";
   } else {
-    setTimeout(function(){cleanUp(zip);},2000);
+    setTimeout(cleanUpBackup,2000);
   }
 }
+
 function backup() {
-  $('input[value="_(Boot device backup)_"]').val('_(Creating boot device backup)_...').prop('disabled',true);
-  $('div.spinner').show('slow');
-  $('#pleaseWait').show('slow');
-  $.post('/webGui/include/Download.php',{cmd:'backup'},function(zip) {
-    if (zip) {
-      location = '/'+zip;
-      setTimeout(function(){cleanUp(zip);},6000);
-    } else {
-      var $btn = $('input[value="_(Creating boot device backup)_..."]');
-      $btn.val("_(Boot device backup)_").prop('disabled',false);
-      $('div.spinner').hide('slow');
-      $('#pleaseWait').hide('slow');
-      swal({title:"_(Creation error)_",text:"_(Insufficient free disk space available)_",type:'error',html:true,confirmButtonText:"_(Ok)_"});
-    }
+  backupFile = "";
+  var level = $('#backupLevel').val();
+  nchan_backup.start();
+  $.post('/webGui/include/StartCommand.php',{cmd:'flash_backup '+level+' nchan',start:0},function() {
+    swal({title:"_(Boot device backup)_ - <span id='pluginProgressTitle'>_(In Progress)_ <i class='fa fa-refresh fa-spin'></i></span>",text:"<pre id='swaltext'></pre><hr>",html:true,animation:'none',showConfirmButton:true,confirmButtonText:"_(Close)_"},function(){
+      nchan_backup.stop();
+      $('.sweet-alert').hide('fast').removeClass('nchan');
+    });
+    $('.sweet-alert').addClass('nchan');
+    $('button.confirm').prop('disabled',true);
   });
 }
 </script>
-<div id="pleaseWait" style="display:none;text-align:center;margin-bottom:24px"><span class="red-text strong">_(Please wait)_... _(creating boot device backup zip file (this may take several minutes))_</span></div>
-
 _(Active GUID)_:
 : <?if (!empty($var['flashGUID'])):?>
     <?=htmlspecialchars($var['flashGUID'], ENT_QUOTES, 'UTF-8');?>
@@ -81,6 +98,14 @@ _(TPM GUID)_:
 
 :flash_backup_help:
 
+_(Compression level)_:
+: <select id="backupLevel">
+    <option value="0">_(None - fastest)_</option>
+    <option value="1">_(Low)_</option>
+    <option value="6" selected>_(Normal)_</option>
+    <option value="9">_(Maximum - smallest)_</option>
+  </select>
+
 &nbsp;
 : <span class="inline-block">
     <input type="button" value="_(Boot device backup)_" onclick="backup()">

emhttp/plugins/dynamix/include/Download.php

@@ -49,8 +49,5 @@ function validpath($file) {
   if ($backup = readlink("$docroot/$file")) unlink($backup);
   @unlink("$docroot/$file");
   break;
-case 'backup':
-  echo exec("$docroot/webGui/scripts/flash_backup");
-  break;
 }
 ?>

emhttp/plugins/dynamix/scripts/flash_backup

@@ -1,6 +1,6 @@
 #!/usr/bin/php -q
 <?PHP
-/* Copyright 2005-2023, Lime Technology
+/* Copyright 2005-2025, Lime Technology
  * Copyright 2012-2023, Bergware International.
  *
  * This program is free software; you can redistribute it and/or
@@ -12,38 +12,118 @@
  */
 ?>
 <?
+/* Create a backup zip of the boot device (USB flash).
+ *
+ * Usage: flash_backup [compression-level] [nchan]
+ *   compression-level  0 (store, fastest) .. 9 (smallest, slowest), default 6
+ *   nchan              trailing marker added by StartCommand.php; ignored here
+ *
+ * Launched as a tracked background job via StartCommand.php (same mechanism as
+ * plugin/docker operations), so it gets a PID, run de-duplication and abort for
+ * free. Progress is streamed to the 'flash_backup' nchan channel using the
+ * standard _DONE_/_ERROR_ sentinels; a _FILE_<name> message tells the GUI which
+ * archive to download. The work runs under nice + best-effort-low ionice and
+ * uses the streaming `zip` binary, so it stays gentle on CPU, disk and RAM
+ * even on large flash drives.
+ */
 $docroot ??= ($_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp');
 require_once "$docroot/webGui/include/Wrappers.php";
+require_once "$docroot/webGui/include/publish.php";
+require_once "$docroot/webGui/include/Helpers.php"; // pools_filter()
+
+// This runs as a CLI background job (launched via StartCommand.php), where the
+// GUI translation layer is not loaded. Provide a no-op _() fallback like other
+// CLI scripts (e.g. monitor) so progress strings work; they stay in English.
+if (!function_exists('_')) { function _($text) { return $text; } }
+
+$channel = 'flash_backup';
+
+// Stream a progress line to the GUI over nchan.
+function write(...$messages) {
+  global $channel;
+  foreach ($messages as $message) publish($channel, $message, 1, false);
+}
 
 $var = (array)@parse_ini_file('/var/local/emhttp/var.ini');
+$disks = (array)@parse_ini_file('/var/local/emhttp/disks.ini', true);
+$pools = pools_filter($disks);
 $dir = ['system','appdata','isos','domains'];
 $out = ['prev','previous'];
 
+// Clamp the requested compression level to a valid zip range (default 6).
+$level = isset($argv[1]) && is_numeric($argv[1]) ? (int)$argv[1] : 6;
+$level = max(0, min(9, $level));
+
 $server = isset($var['NAME']) ? str_replace(' ','_',strtolower($var['NAME'])) : 'tower';
 $mydate = date('Ymd-Hi');
 $osVersion = _var($var,'version','_unknown');
 $backup = "$server-v$osVersion-boot-backup-$mydate.zip";
 
+write(_('Preparing backup')."...");
+
+// Estimate required space (1.5x current flash usage) and pick a target, in
+// order: a cache/SSD pool first (fast, and writing straight to the pool mount
+// bypasses the slower /mnt/user FUSE layer), then the RAM-backed rootfs, then an
+// array-backed user share. This keeps it fast on typical systems while avoiding
+// holding a large archive in RAM unless nothing else fits.
 $used = exec("df /boot|awk 'END{print $3}'") * 1.5;
-$free = exec("df /|awk 'END{print $4}'");
-if ($free > $used) $zip = "/$backup"; else {
+$zip = null;
+foreach ($pools as $pool) {
+  if (!is_dir("/mnt/$pool")) continue;
+  $free = exec("df ".escapeshellarg("/mnt/$pool")."|awk 'END{print $4}'");
+  if ($free > $used) {$zip = "/mnt/$pool/$backup"; break;}
+}
+if (!$zip) {
+  $free = exec("df /|awk 'END{print $4}'");
+  if ($free > $used) $zip = "/$backup";
+}
+if (!$zip) {
   foreach ($dir as $share) {
     if (!is_dir("/mnt/user/$share")) continue;
-    $free = exec("df /mnt/user/$share|awk 'END{print $4}'");
+    $free = exec("df ".escapeshellarg("/mnt/user/$share")."|awk 'END{print $4}'");
     if ($free > $used) {$zip = "/mnt/user/$share/$backup"; break;}
   }
 }
-if (isset($zip)) {
-  chdir("/boot");
-  foreach (glob("*",GLOB_NOSORT+GLOB_ONLYDIR) as $folder) {
-    if (in_array($folder,$out)) continue;
-    exec("zip -qr ".escapeshellarg($zip)." ".escapeshellarg($folder));
-  }
-  foreach (glob("*",GLOB_NOSORT) as $file) {
-    if (is_dir($file)) continue;
-    exec("zip -q ".escapeshellarg($zip)." ".escapeshellarg($file));
-  }
-  symlink($zip,"$docroot/$backup");
-  echo $backup;
+
+if (!$zip) {
+  write(_('Insufficient free disk space available'));
+  write('_ERROR_');
+  exit(1);
+}
+
+// Collect top-level entries to archive, skipping previous backups.
+chdir('/boot');
+$items = [];
+foreach (glob("*",GLOB_NOSORT) as $entry) {
+  if (in_array($entry,$out)) continue;
+  $items[] = $entry;
 }
+$total = count($items);
+
+// Run zip at low CPU/IO priority so the backup stays gentle without being
+// starved: best-effort IO at the lowest priority (not the idle class, which can
+// throttle throughput to a crawl on busy/ZFS systems).
+$prio = 'nice -n 19 ionice -c2 -n7';
+$ok = true;
+foreach ($items as $i => $entry) {
+  $pct = $total ? round(($i/$total)*100) : 0;
+  write(sprintf(_('Compressing %s')." (%d/%d - %d%%)", $entry, $i+1, $total, $pct));
+  $flag = is_dir($entry) ? '-r' : '';
+  exec("$prio zip -$level -q $flag ".escapeshellarg($zip)." ".escapeshellarg($entry)." 2>/dev/null", $o, $rc);
+  // zip rc 12 = "nothing to do"; treat any other non-zero result as a failure.
+  if ($rc !== 0 && $rc !== 12) $ok = false;
+}
+
+if (!$ok || !is_file($zip)) {
+  @unlink($zip);
+  write(_('Backup failed'));
+  write('_ERROR_');
+  exit(1);
+}
+
+// Expose the archive to the browser via a symlink in the webGui docroot.
+symlink($zip,"$docroot/$backup");
+write(_('Backup complete')."...");
+write("_FILE_$backup");
+write('_DONE_');
 ?>