feat(flash-backup): streaming download + save-to-server, with compression & progress

emhttp/languages/en_US/helptext.txt

@@ -442,7 +442,9 @@ array or the cache disk/pool.
 :end
 
 :flash_backup_help:
-Use *Boot device backup* to create a single zip file of the current contents of the boot device and store it locally on your computer.
+Use *Boot device backup* to create a single zip file of the current contents of the boot device and download it to your computer.
+
+The archive is streamed directly to your browser as it is built, so the download begins shortly after you start it and nothing is stored on the server. Choose a *Compression level* first: *None* is the fastest but produces the largest file, while *Maximum* is the slowest but produces the smallest file. The previous OS release and other non-essential files are excluded to keep the backup small.
 :end
 
 :syslinux_cfg_help:

emhttp/plugins/dynamix/BootInfo.page

@@ -16,36 +16,35 @@ Tag="usb"
 ?>
 <?$tpmEnabled = !empty($var['flashGUID']) && strpos($var['flashGUID'], '01-') !== 0 && !empty($var['tpmGUID']) ? true : false;?>
 <script>
-function cleanUp(zip) {
-  if (document.hasFocus()) {
-    $('input[value="_(Creating boot device backup)_..."]').val("_(Boot device backup)_").prop('disabled',false);
-    $('div.spinner').hide('slow');
-    $('#pleaseWait').hide('slow');
-    $.post('/webGui/include/Download.php',{cmd:'unlink',file:zip});
-  } else {
-    setTimeout(function(){cleanUp(zip);},2000);
+// The backup is streamed straight to the browser as it is built - no staging
+// file, no size limit, and it works with the array stopped. The download itself
+// has no known size (it is compressed on the fly), so the server reports
+// progress over the 'flash_backup' nchan channel and we show a real bar.
+var nchan_backup = new NchanSubscriber('/sub/flash_backup',{subscriber:'websocket'});
+nchan_backup.on('message', function(data) {
+  if (data == '_DONE_') {
+    setBackupProgress(100);
+    nchan_backup.stop();
+    setTimeout(function(){ swal.close(); }, 1200);
+    return;
   }
+  var pct = parseInt(data, 10);
+  if (!isNaN(pct)) setBackupProgress(pct);
+});
+
+function setBackupProgress(pct) {
+  $('#fbBar').css('width', pct + '%');
+  $('#fbPct').text(pct + '%');
 }
+
 function backup() {
-  $('input[value="_(Boot device backup)_"]').val('_(Creating boot device backup)_...').prop('disabled',true);
-  $('div.spinner').show('slow');
-  $('#pleaseWait').show('slow');
-  $.post('/webGui/include/Download.php',{cmd:'backup'},function(zip) {
-    if (zip) {
-      location = '/'+zip;
-      setTimeout(function(){cleanUp(zip);},6000);
-    } else {
-      var $btn = $('input[value="_(Creating boot device backup)_..."]');
-      $btn.val("_(Boot device backup)_").prop('disabled',false);
-      $('div.spinner').hide('slow');
-      $('#pleaseWait').hide('slow');
-      swal({title:"_(Creation error)_",text:"_(Insufficient free disk space available)_",type:'error',html:true,confirmButtonText:"_(Ok)_"});
-    }
-  });
+  var level = $('#backupLevel').val();
+  nchan_backup.start();
+  swal({title:"_(Boot device backup)_",text:"<div style='background:var(--border-color,#ccc);border-radius:3px;overflow:hidden;height:18px;margin:10px 0'><div id='fbBar' style='height:100%;width:0%;background:#ff8c2f;transition:width .3s'></div></div><div id='fbPct'>0%</div><div style='margin-top:8px'>_(Your download will begin shortly)_</div>",html:true,animation:'none',showConfirmButton:false});
+  // Attachment response keeps us on this page while the browser downloads.
+  window.location = '/webGui/include/FlashBackup.php?level='+level;
 }
 </script>
-<div id="pleaseWait" style="display:none;text-align:center;margin-bottom:24px"><span class="red-text strong">_(Please wait)_... _(creating boot device backup zip file (this may take several minutes))_</span></div>
-
 _(Active GUID)_:
 : <?if (!empty($var['flashGUID'])):?>
     <?=htmlspecialchars($var['flashGUID'], ENT_QUOTES, 'UTF-8');?>
@@ -81,6 +80,14 @@ _(TPM GUID)_:
 
 :flash_backup_help:
 
+_(Compression level)_:
+: <select id="backupLevel">
+    <option value="0">_(None - fastest)_</option>
+    <option value="1">_(Low)_</option>
+    <option value="6" selected>_(Normal)_</option>
+    <option value="9">_(Maximum - smallest)_</option>
+  </select>
+
 &nbsp;
 : <span class="inline-block">
     <input type="button" value="_(Boot device backup)_" onclick="backup()">

emhttp/plugins/dynamix/include/Download.php

@@ -49,8 +49,5 @@ function validpath($file) {
   if ($backup = readlink("$docroot/$file")) unlink($backup);
   @unlink("$docroot/$file");
   break;
-case 'backup':
-  echo exec("$docroot/webGui/scripts/flash_backup");
-  break;
 }
 ?>

emhttp/plugins/dynamix/include/FlashBackup.php

@@ -0,0 +1,71 @@
+<?PHP
+/* Copyright 2005-2025, Lime Technology
+ * Copyright 2012-2023, Bergware International.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version 2,
+ * as published by the Free Software Foundation.
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ */
+?>
+<?
+/* Stream a boot-device (flash) backup straight to the browser as it is built,
+ * so nothing is ever staged on disk or in RAM. The zip is produced by
+ * webGui/scripts/flash_backup (writing to stdout); we pipe it to the client and,
+ * as bytes flow, publish a percentage to the 'flash_backup' nchan channel so the
+ * GUI can show a real progress bar. The percentage is bytes-sent over the total
+ * flash size reported by `flash_backup size` (the boot device is mostly already-
+ * compressed OS files, so output closely tracks input - the bar is accurate in
+ * practice and we cap it at 99% until the stream actually finishes).
+ *
+ * Query params:
+ *   level   zip compression level 0..9 (default 6)
+ */
+$docroot ??= ($_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp');
+require_once "$docroot/webGui/include/publish.php";
+
+$channel = 'flash_backup';
+$script = "$docroot/webGui/scripts/flash_backup";
+
+$var = (array)@parse_ini_file('/var/local/emhttp/var.ini');
+$level = isset($_GET['level']) && is_numeric($_GET['level']) ? max(0, min(9, (int)$_GET['level'])) : 6;
+
+$server = isset($var['NAME']) ? str_replace(' ', '_', strtolower($var['NAME'])) : 'tower';
+$osVersion = $var['version'] ?? 'unknown';
+$name = "$server-v$osVersion-boot-backup-".date('Ymd-Hi').".zip";
+
+// Total size of the included set, used as the progress denominator.
+$total = (int)trim(shell_exec(escapeshellarg($script).' size 2>/dev/null'));
+
+// Stream for as long as it takes, and let zip die if the user cancels.
+set_time_limit(0);
+ignore_user_abort(false);
+
+header('Content-Type: application/zip');
+header('Content-Disposition: attachment; filename="'.$name.'"');
+header('X-Accel-Buffering: no'); // tell nginx to stream, not spool to a temp file
+header('Cache-Control: no-store');
+while (ob_get_level()) ob_end_clean();
+
+$h = popen(escapeshellarg($script).' '.(int)$level.' 2>/dev/null', 'r');
+if ($h) {
+  $sent = 0; $last = 0;
+  while (!feof($h)) {
+    $buf = fread($h, 1 << 18);
+    if ($buf === '' || $buf === false) break;
+    echo $buf;
+    flush();
+    $sent += strlen($buf);
+    $now = microtime(true);
+    if ($total > 0 && $now - $last > 0.4) {
+      publish($channel, (string)min(99, intdiv($sent * 100, $total)), 1, false);
+      $last = $now;
+    }
+  }
+  pclose($h);
+}
+publish($channel, '_DONE_', 1, false);
+exit;
+?>

emhttp/plugins/dynamix/scripts/flash_backup

@@ -1,6 +1,6 @@
 #!/usr/bin/php -q
 <?PHP
-/* Copyright 2005-2023, Lime Technology
+/* Copyright 2005-2025, Lime Technology
  * Copyright 2012-2023, Bergware International.
  *
  * This program is free software; you can redistribute it and/or
@@ -12,38 +12,50 @@
  */
 ?>
 <?
-$docroot ??= ($_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp');
-require_once "$docroot/webGui/include/Wrappers.php";
+/* Stream a zip backup of the boot device (USB flash) to stdout.
+ *
+ * Usage: flash_backup [compression-level] > backup.zip   (stream the archive)
+ *        flash_backup size                                (print total bytes)
+ *   compression-level  0 (store, fastest) .. 9 (smallest, slowest), default 6
+ *
+ * The archive is written straight to stdout so it can be streamed directly to a
+ * browser (see include/FlashBackup.php) without ever staging a file on disk or
+ * in RAM - this avoids any size limit, works with the array stopped, and needs
+ * no cleanup. zip runs under nice + best-effort-low ionice to stay gentle on
+ * CPU and disk. The prior-release dirs (previous/prev) and desktop junk are
+ * excluded so the archive stays small.
+ *
+ * The 'size' mode prints the total byte size of the same included set; the
+ * endpoint uses it as the denominator for a download progress bar.
+ */
 
-$var = (array)@parse_ini_file('/var/local/emhttp/var.ini');
-$dir = ['system','appdata','isos','domains'];
-$out = ['prev','previous'];
+$sizeOnly = isset($argv[1]) && $argv[1] === 'size';
+// Clamp the requested compression level to a valid zip range (default 6).
+$level = isset($argv[1]) && is_numeric($argv[1]) ? max(0, min(9, (int)$argv[1])) : 6;
 
-$server = isset($var['NAME']) ? str_replace(' ','_',strtolower($var['NAME'])) : 'tower';
-$mydate = date('Ymd-Hi');
-$osVersion = _var($var,'version','_unknown');
-$backup = "$server-v$osVersion-boot-backup-$mydate.zip";
+// Top-level entries to exclude. 'previous'/'prev' hold the entire prior OS
+// release (often >1GB); the rest is desktop junk created when the flash is
+// mounted on a Mac/PC.
+$out = ['prev','previous','System Volume Information','.Trash-0','.Trashes','.Spotlight-V100','.fseventsd','.TemporaryItems'];
 
-$used = exec("df /boot|awk 'END{print $3}'") * 1.5;
-$free = exec("df /|awk 'END{print $4}'");
-if ($free > $used) $zip = "/$backup"; else {
-  foreach ($dir as $share) {
-    if (!is_dir("/mnt/user/$share")) continue;
-    $free = exec("df /mnt/user/$share|awk 'END{print $4}'");
-    if ($free > $used) {$zip = "/mnt/user/$share/$backup"; break;}
-  }
+chdir('/boot');
+$items = [];
+foreach (glob('*', GLOB_NOSORT) as $entry) {
+  if (in_array($entry, $out)) continue;
+  if (preg_match('/-boot-backup-.*\.zip$/', $entry)) continue; // stray old backups
+  $items[] = $entry;
 }
-if (isset($zip)) {
-  chdir("/boot");
-  foreach (glob("*",GLOB_NOSORT+GLOB_ONLYDIR) as $folder) {
-    if (in_array($folder,$out)) continue;
-    exec("zip -qr ".escapeshellarg($zip)." ".escapeshellarg($folder));
-  }
-  foreach (glob("*",GLOB_NOSORT) as $file) {
-    if (is_dir($file)) continue;
-    exec("zip -q ".escapeshellarg($zip)." ".escapeshellarg($file));
-  }
-  symlink($zip,"$docroot/$backup");
-  echo $backup;
+if (!$items) exit(1);
+$args = implode(' ', array_map('escapeshellarg', $items));
+
+// 'size' mode: print the total apparent byte size of the included set and exit.
+if ($sizeOnly) {
+  echo (int)trim(exec("du -sbc $args 2>/dev/null | awk 'END{print $1}'"));
+  exit(0);
 }
+
+// Stream the archive to stdout. zip writes a valid streamable archive (data
+// descriptors) when its output is not seekable.
+passthru("nice -n 19 ionice -c2 -n7 zip -$level -qr - $args 2>/dev/null", $rc);
+exit($rc);
 ?>