From 6094f9890bd47f80a6774866bfc1d73bd1b08731 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Mon, 22 Jun 2026 15:42:57 -0400 Subject: [PATCH 01/16] feat(flash-backup): background boot backup with compression level Run the boot device (USB flash) backup as a detached background task instead of blocking the HTTP request for the entire zip creation. - Add a compression-level selector (None/Low/Normal/Maximum) on the Boot Device page; the chosen zip level (0/1/6/9) is passed to flash_backup. - Stream live progress over a new 'flash_backup' nchan channel and trigger the download automatically on completion (mirrors the Diagnostics flow). - Run zip under nice/ionice so the backup stays gentle on CPU and disk. - Prefer a real disk (array/pool share) over the RAM-backed rootfs for the archive so a large backup does not consume system memory; fall back to / only when no share has room. Co-Authored-By: Claude Opus 4.8 --- emhttp/languages/en_US/helptext.txt | 2 + emhttp/plugins/dynamix/BootInfo.page | 75 +++++++++++----- emhttp/plugins/dynamix/include/Download.php | 6 +- emhttp/plugins/dynamix/scripts/flash_backup | 98 ++++++++++++++++----- 4 files changed, 136 insertions(+), 45 deletions(-) diff --git a/emhttp/languages/en_US/helptext.txt b/emhttp/languages/en_US/helptext.txt index 92ee679a46..ba77faa2de 100644 --- a/emhttp/languages/en_US/helptext.txt +++ b/emhttp/languages/en_US/helptext.txt @@ -443,6 +443,8 @@ array or the cache disk/pool. :flash_backup_help: Use *Boot device backup* to create a single zip file of the current contents of the boot device and store it locally on your computer. + +The backup runs in the background and reports its progress, then downloads automatically when finished. Choose a *Compression level* before starting: *None* is the fastest but produces the largest file, while *Maximum* is the slowest but produces the smallest file. :end :syslinux_cfg_help: diff --git a/emhttp/plugins/dynamix/BootInfo.page b/emhttp/plugins/dynamix/BootInfo.page index fd39f20f88..875b1e5b76 100644 --- a/emhttp/plugins/dynamix/BootInfo.page +++ b/emhttp/plugins/dynamix/BootInfo.page @@ -16,36 +16,55 @@ Tag="usb" ?> - - _(Active GUID)_: : @@ -81,8 +100,16 @@ _(TPM GUID)_: :flash_backup_help: +_(Compression level)_: +: +   : - + diff --git a/emhttp/plugins/dynamix/include/Download.php b/emhttp/plugins/dynamix/include/Download.php index e7146479a4..7954ea3a0c 100644 --- a/emhttp/plugins/dynamix/include/Download.php +++ b/emhttp/plugins/dynamix/include/Download.php @@ -50,7 +50,11 @@ function validpath($file) { @unlink("$docroot/$file"); break; case 'backup': - echo exec("$docroot/webGui/scripts/flash_backup"); + // Run the flash backup detached; progress is streamed over the + // 'flash_backup' nchan channel and the GUI triggers the download on _DONE_. + $level = isset($_POST['level']) && is_numeric($_POST['level']) ? (int)$_POST['level'] : 6; + $level = max(0, min(9, $level)); + exec("nohup $docroot/webGui/scripts/flash_backup $level 1>/dev/null 2>&1 &"); break; } ?> diff --git a/emhttp/plugins/dynamix/scripts/flash_backup b/emhttp/plugins/dynamix/scripts/flash_backup index e610e19d2a..9216764666 100755 --- a/emhttp/plugins/dynamix/scripts/flash_backup +++ b/emhttp/plugins/dynamix/scripts/flash_backup @@ -1,6 +1,6 @@ #!/usr/bin/php -q $used) $zip = "/$backup"; else { - foreach ($dir as $share) { - if (!is_dir("/mnt/user/$share")) continue; - $free = exec("df /mnt/user/$share|awk 'END{print $4}'"); - if ($free > $used) {$zip = "/mnt/user/$share/$backup"; break;} - } +$zip = null; +foreach ($dir as $share) { + if (!is_dir("/mnt/user/$share")) continue; + $free = exec("df /mnt/user/$share|awk 'END{print $4}'"); + if ($free > $used) {$zip = "/mnt/user/$share/$backup"; break;} } -if (isset($zip)) { - chdir("/boot"); - foreach (glob("*",GLOB_NOSORT+GLOB_ONLYDIR) as $folder) { - if (in_array($folder,$out)) continue; - exec("zip -qr ".escapeshellarg($zip)." ".escapeshellarg($folder)); - } - foreach (glob("*",GLOB_NOSORT) as $file) { - if (is_dir($file)) continue; - exec("zip -q ".escapeshellarg($zip)." ".escapeshellarg($file)); - } - symlink($zip,"$docroot/$backup"); - echo $backup; +if (!$zip) { + $free = exec("df /|awk 'END{print $4}'"); + if ($free > $used) $zip = "/$backup"; } + +if (!$zip) { + write("_ERROR_"._('Insufficient free disk space available')); + write('_DONE_'); + exit(1); +} + +// Collect top-level entries to archive, skipping previous backups. +chdir('/boot'); +$items = []; +foreach (glob("*",GLOB_NOSORT) as $entry) { + if (in_array($entry,$out)) continue; + $items[] = $entry; +} +$total = count($items); + +// Run zip with low CPU/IO priority so the backup stays responsive under load. +$prio = 'nice -n 19 ionice -c3'; +$ok = true; +foreach ($items as $i => $entry) { + $pct = $total ? round(($i/$total)*100) : 0; + write(sprintf(_('Compressing %s')." (%d/%d - %d%%)", $entry, $i+1, $total, $pct)); + $flag = is_dir($entry) ? '-r' : ''; + exec("$prio zip -$level -q $flag ".escapeshellarg($zip)." ".escapeshellarg($entry)." 2>/dev/null", $o, $rc); + // zip rc 12 = "nothing to do"; treat any other non-zero result as a failure. + if ($rc !== 0 && $rc !== 12) $ok = false; +} + +if (!$ok || !is_file($zip)) { + @unlink($zip); + write("_ERROR_"._('Backup failed')); + write('_DONE_'); + exit(1); +} + +// Expose the archive to the browser via a symlink in the webGui docroot. +symlink($zip,"$docroot/$backup"); +write(_('Backup complete')."..."); +write("_FILE_$backup"); +write('_DONE_'); ?> From d3f79545655ebdcaf00ba2e68dd4e924fcb74d4d Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Mon, 22 Jun 2026 16:32:41 -0400 Subject: [PATCH 02/16] refactor(flash-backup): use StartCommand.php background-job + standard nchan window Switch the boot backup from a bespoke modal to the same background-job mechanism plugin/docker operations use: - Launch via StartCommand.php (tracked PID, run de-duplication, abortable) instead of a one-off nohup in Download.php. - Stream into the standard progress window (pre#swaltext / pluginProgressTitle) and reuse the shared openDone()/openError() sentinel handlers; the script now emits the bare _DONE_/_ERROR_ sentinels they expect. - Keep the _FILE_ message so the GUI still auto-downloads the archive on completion, then cleans it up. - Drop the now-unused cmd=backup branch from Download.php. Co-Authored-By: Claude Opus 4.8 --- emhttp/plugins/dynamix/BootInfo.page | 52 ++++++++++----------- emhttp/plugins/dynamix/include/Download.php | 7 --- emhttp/plugins/dynamix/scripts/flash_backup | 23 +++++---- 3 files changed, 38 insertions(+), 44 deletions(-) diff --git a/emhttp/plugins/dynamix/BootInfo.page b/emhttp/plugins/dynamix/BootInfo.page index 875b1e5b76..351158ac0d 100644 --- a/emhttp/plugins/dynamix/BootInfo.page +++ b/emhttp/plugins/dynamix/BootInfo.page @@ -16,53 +16,51 @@ Tag="usb" ?> _(Active GUID)_: @@ -110,6 +108,6 @@ _(Compression level)_:   : - + diff --git a/emhttp/plugins/dynamix/include/Download.php b/emhttp/plugins/dynamix/include/Download.php index 7954ea3a0c..bff0860a34 100644 --- a/emhttp/plugins/dynamix/include/Download.php +++ b/emhttp/plugins/dynamix/include/Download.php @@ -49,12 +49,5 @@ function validpath($file) { if ($backup = readlink("$docroot/$file")) unlink($backup); @unlink("$docroot/$file"); break; -case 'backup': - // Run the flash backup detached; progress is streamed over the - // 'flash_backup' nchan channel and the GUI triggers the download on _DONE_. - $level = isset($_POST['level']) && is_numeric($_POST['level']) ? (int)$_POST['level'] : 6; - $level = max(0, min(9, $level)); - exec("nohup $docroot/webGui/scripts/flash_backup $level 1>/dev/null 2>&1 &"); - break; } ?> diff --git a/emhttp/plugins/dynamix/scripts/flash_backup b/emhttp/plugins/dynamix/scripts/flash_backup index 9216764666..6c39205964 100755 --- a/emhttp/plugins/dynamix/scripts/flash_backup +++ b/emhttp/plugins/dynamix/scripts/flash_backup @@ -14,14 +14,17 @@ message tells the GUI which + * archive to download. The work runs under nice/ionice and uses the streaming + * `zip` binary, so it stays gentle on CPU, disk and RAM even on large flash + * drives. */ $docroot ??= ($_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp'); require_once "$docroot/webGui/include/Wrappers.php"; @@ -66,8 +69,8 @@ if (!$zip) { } if (!$zip) { - write("_ERROR_"._('Insufficient free disk space available')); - write('_DONE_'); + write(_('Insufficient free disk space available')); + write('_ERROR_'); exit(1); } @@ -94,8 +97,8 @@ foreach ($items as $i => $entry) { if (!$ok || !is_file($zip)) { @unlink($zip); - write("_ERROR_"._('Backup failed')); - write('_DONE_'); + write(_('Backup failed')); + write('_ERROR_'); exit(1); } From d3f4a7f4ae30d6aa6e566896ad8bd96a361fcfa9 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Tue, 23 Jun 2026 05:31:00 -0400 Subject: [PATCH 03/16] fix(flash-backup): CLI translation fallback, faster IO class, pool-first target Issues found by running the job end-to-end on a live server: - Fatal "Call to undefined function _()": the job runs under php-cli (launched via StartCommand.php) where the GUI translation layer isn't loaded, so every _() call aborted the script immediately. Add a no-op _() fallback like other CLI scripts (monitor, InstallKey). This broke the feature entirely. - ionice -c3 (idle) could starve throughput to a crawl; use best-effort lowest priority (-c2 -n7) instead. (Note: ionice is a no-op on ZFS, which has its own scheduler.) - Storage target now prefers a cache/SSD pool, then RAM, then an array share - and writes straight to the pool mount, bypassing the slower /mnt/user FUSE layer. The previous disk-share-first choice routed through FUSE onto array/ZFS and was very slow; this keeps it fast on typical systems while still avoiding a large archive in RAM unless nothing else fits. Verified on a live 7.3.2 box: archive builds, integrity OK, prev/previous excluded, progress streams over nchan, _FILE_/_DONE_ drive the download. Co-Authored-By: Claude Opus 4.8 --- emhttp/plugins/dynamix/scripts/flash_backup | 43 +++++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/emhttp/plugins/dynamix/scripts/flash_backup b/emhttp/plugins/dynamix/scripts/flash_backup index 6c39205964..b13b2ef227 100755 --- a/emhttp/plugins/dynamix/scripts/flash_backup +++ b/emhttp/plugins/dynamix/scripts/flash_backup @@ -22,13 +22,19 @@ * plugin/docker operations), so it gets a PID, run de-duplication and abort for * free. Progress is streamed to the 'flash_backup' nchan channel using the * standard _DONE_/_ERROR_ sentinels; a _FILE_ message tells the GUI which - * archive to download. The work runs under nice/ionice and uses the streaming - * `zip` binary, so it stays gentle on CPU, disk and RAM even on large flash - * drives. + * archive to download. The work runs under nice + best-effort-low ionice and + * uses the streaming `zip` binary, so it stays gentle on CPU, disk and RAM + * even on large flash drives. */ $docroot ??= ($_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp'); require_once "$docroot/webGui/include/Wrappers.php"; require_once "$docroot/webGui/include/publish.php"; +require_once "$docroot/webGui/include/Helpers.php"; // pools_filter() + +// This runs as a CLI background job (launched via StartCommand.php), where the +// GUI translation layer is not loaded. Provide a no-op _() fallback like other +// CLI scripts (e.g. monitor) so progress strings work; they stay in English. +if (!function_exists('_')) { function _($text) { return $text; } } $channel = 'flash_backup'; @@ -39,6 +45,8 @@ function write(...$messages) { } $var = (array)@parse_ini_file('/var/local/emhttp/var.ini'); +$disks = (array)@parse_ini_file('/var/local/emhttp/disks.ini', true); +$pools = pools_filter($disks); $dir = ['system','appdata','isos','domains']; $out = ['prev','previous']; @@ -53,20 +61,29 @@ $backup = "$server-v$osVersion-boot-backup-$mydate.zip"; write(_('Preparing backup')."..."); -// Estimate required space (1.5x current flash usage) and pick a target. -// Prefer real disk (array/pool shares) over the RAM-backed rootfs so a large -// backup does not consume system memory; fall back to / only when no share fits. +// Estimate required space (1.5x current flash usage) and pick a target, in +// order: a cache/SSD pool first (fast, and writing straight to the pool mount +// bypasses the slower /mnt/user FUSE layer), then the RAM-backed rootfs, then an +// array-backed user share. This keeps it fast on typical systems while avoiding +// holding a large archive in RAM unless nothing else fits. $used = exec("df /boot|awk 'END{print $3}'") * 1.5; $zip = null; -foreach ($dir as $share) { - if (!is_dir("/mnt/user/$share")) continue; - $free = exec("df /mnt/user/$share|awk 'END{print $4}'"); - if ($free > $used) {$zip = "/mnt/user/$share/$backup"; break;} +foreach ($pools as $pool) { + if (!is_dir("/mnt/$pool")) continue; + $free = exec("df ".escapeshellarg("/mnt/$pool")."|awk 'END{print $4}'"); + if ($free > $used) {$zip = "/mnt/$pool/$backup"; break;} } if (!$zip) { $free = exec("df /|awk 'END{print $4}'"); if ($free > $used) $zip = "/$backup"; } +if (!$zip) { + foreach ($dir as $share) { + if (!is_dir("/mnt/user/$share")) continue; + $free = exec("df ".escapeshellarg("/mnt/user/$share")."|awk 'END{print $4}'"); + if ($free > $used) {$zip = "/mnt/user/$share/$backup"; break;} + } +} if (!$zip) { write(_('Insufficient free disk space available')); @@ -83,8 +100,10 @@ foreach (glob("*",GLOB_NOSORT) as $entry) { } $total = count($items); -// Run zip with low CPU/IO priority so the backup stays responsive under load. -$prio = 'nice -n 19 ionice -c3'; +// Run zip at low CPU/IO priority so the backup stays gentle without being +// starved: best-effort IO at the lowest priority (not the idle class, which can +// throttle throughput to a crawl on busy/ZFS systems). +$prio = 'nice -n 19 ionice -c2 -n7'; $ok = true; foreach ($items as $i => $entry) { $pct = $total ? round(($i/$total)*100) : 0; From c90b023e555f4b52f5838855b65adfade9b2d7b2 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Tue, 23 Jun 2026 05:46:57 -0400 Subject: [PATCH 04/16] fix(flash-backup): stage archive in RAM first so it works with the array stopped The backup archive is a throwaway staging file (built, downloaded, deleted), so the destination should be the fastest always-available location. Restore the original RAM-first behavior: rootfs (always mounted, even with the array stopped) -> cache/SSD pool -> array share. Pools only mount with the array, so pool-first failed to deliver a backup when the array was stopped. Disk tiers are kept only for flashes too large to stage in RAM. Process memory stays low regardless via the streaming zip binary. Co-Authored-By: Claude Opus 4.8 --- emhttp/plugins/dynamix/scripts/flash_backup | 33 +++++++++++---------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/emhttp/plugins/dynamix/scripts/flash_backup b/emhttp/plugins/dynamix/scripts/flash_backup index b13b2ef227..09d7173034 100755 --- a/emhttp/plugins/dynamix/scripts/flash_backup +++ b/emhttp/plugins/dynamix/scripts/flash_backup @@ -62,27 +62,30 @@ $backup = "$server-v$osVersion-boot-backup-$mydate.zip"; write(_('Preparing backup')."..."); // Estimate required space (1.5x current flash usage) and pick a target, in -// order: a cache/SSD pool first (fast, and writing straight to the pool mount -// bypasses the slower /mnt/user FUSE layer), then the RAM-backed rootfs, then an -// array-backed user share. This keeps it fast on typical systems while avoiding -// holding a large archive in RAM unless nothing else fits. +// order: the RAM-backed rootfs first, then a cache/SSD pool, then an array +// share. The archive is a throwaway staging file - built here, downloaded by +// the browser, then deleted - so RAM is the right home: it is always mounted +// (works even with the array stopped) and fastest. We only spill to disk when +// the flash is too large to stage in RAM with comfortable headroom. (Keeping +// the *process* RAM-light is handled separately by streaming via the zip +// binary rather than buffering the archive in memory.) $used = exec("df /boot|awk 'END{print $3}'") * 1.5; $zip = null; -foreach ($pools as $pool) { +// 1. RAM-backed rootfs (always available, fastest). +$free = exec("df /|awk 'END{print $4}'"); +if ($free > $used) $zip = "/$backup"; +// 2. Too big for RAM: a cache/SSD pool, written straight to the pool mount to +// bypass the slower /mnt/user FUSE layer. Pools require the array started. +if (!$zip) foreach ($pools as $pool) { if (!is_dir("/mnt/$pool")) continue; $free = exec("df ".escapeshellarg("/mnt/$pool")."|awk 'END{print $4}'"); if ($free > $used) {$zip = "/mnt/$pool/$backup"; break;} } -if (!$zip) { - $free = exec("df /|awk 'END{print $4}'"); - if ($free > $used) $zip = "/$backup"; -} -if (!$zip) { - foreach ($dir as $share) { - if (!is_dir("/mnt/user/$share")) continue; - $free = exec("df ".escapeshellarg("/mnt/user/$share")."|awk 'END{print $4}'"); - if ($free > $used) {$zip = "/mnt/user/$share/$backup"; break;} - } +// 3. Last resort: an array-backed user share. +if (!$zip) foreach ($dir as $share) { + if (!is_dir("/mnt/user/$share")) continue; + $free = exec("df ".escapeshellarg("/mnt/user/$share")."|awk 'END{print $4}'"); + if ($free > $used) {$zip = "/mnt/user/$share/$backup"; break;} } if (!$zip) { From 6bcfefe1b792135a3e257bf0741fe69dd81ea87b Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Tue, 23 Jun 2026 06:06:30 -0400 Subject: [PATCH 05/16] feat(flash-backup): boot-pool fallback, stale-archive reaper, broader excludes - Add the boot device's own pool as a last-resort staging target. /boot is always mounted (even with the array stopped), so this guarantees a target when RAM is full and no pool/array share is available. Gated to real pool filesystems (zfs/btrfs/xfs) so a FAT/USB flash - which is not a pool and can't hold a >4GB file - is never chosen. - Reap any leftover staging archive at the start of each run (readlink the docroot symlink + delete its target), so a copy can't linger - especially in RAM - if the browser-side cleanup didn't fire (e.g. tab closed mid-download). The normal post-download Download.php cmd=unlink still runs. - Exclude more: prior-release dirs already skipped (previous/prev, ~1GB+), now also common desktop junk (System Volume Information, .Trashes, .fseventsd, .Spotlight-V100, .TemporaryItems) and any stray *-boot-backup-*.zip a user left on the flash. Verified on a live ZFS box: previous/ (1.1G) excluded, reaper removes a seeded stale artifact, archive integrity OK. Co-Authored-By: Claude Opus 4.8 --- emhttp/plugins/dynamix/scripts/flash_backup | 37 ++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/emhttp/plugins/dynamix/scripts/flash_backup b/emhttp/plugins/dynamix/scripts/flash_backup index 09d7173034..9485878683 100755 --- a/emhttp/plugins/dynamix/scripts/flash_backup +++ b/emhttp/plugins/dynamix/scripts/flash_backup @@ -24,7 +24,10 @@ * standard _DONE_/_ERROR_ sentinels; a _FILE_ message tells the GUI which * archive to download. The work runs under nice + best-effort-low ionice and * uses the streaming `zip` binary, so it stays gentle on CPU, disk and RAM - * even on large flash drives. + * even on large flash drives. The prior-release dirs (previous/prev) and + * desktop junk are excluded so the archive stays small. The finished zip is a + * throwaway staged in RAM (then pool/array, finally the boot device's own pool) + * and is deleted right after the browser downloads it. */ $docroot ??= ($_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp'); require_once "$docroot/webGui/include/Wrappers.php"; @@ -48,7 +51,10 @@ $var = (array)@parse_ini_file('/var/local/emhttp/var.ini'); $disks = (array)@parse_ini_file('/var/local/emhttp/disks.ini', true); $pools = pools_filter($disks); $dir = ['system','appdata','isos','domains']; -$out = ['prev','previous']; +// Top-level entries to exclude from the backup. 'previous'/'prev' hold the +// entire prior OS release (often >1GB) and must never bloat the archive; +// the rest is desktop junk created when the flash is mounted on a Mac/PC. +$out = ['prev','previous','System Volume Information','.Trash-0','.Trashes','.Spotlight-V100','.fseventsd','.TemporaryItems']; // Clamp the requested compression level to a valid zip range (default 6). $level = isset($argv[1]) && is_numeric($argv[1]) ? (int)$argv[1] : 6; @@ -59,6 +65,16 @@ $mydate = date('Ymd-Hi'); $osVersion = _var($var,'version','_unknown'); $backup = "$server-v$osVersion-boot-backup-$mydate.zip"; +// Safety net: reap any leftover staging archive from a previous run whose +// browser-side cleanup never fired (e.g. the tab was closed mid-download), so a +// stale copy can't linger - especially in RAM. The normal path still deletes +// the archive right after the download via Download.php cmd=unlink. +foreach (glob("$docroot/*-boot-backup-*.zip") as $stale) { + $target = @readlink($stale); + if ($target && is_file($target)) @unlink($target); + @unlink($stale); +} + write(_('Preparing backup')."..."); // Estimate required space (1.5x current flash usage) and pick a target, in @@ -81,12 +97,23 @@ if (!$zip) foreach ($pools as $pool) { $free = exec("df ".escapeshellarg("/mnt/$pool")."|awk 'END{print $4}'"); if ($free > $used) {$zip = "/mnt/$pool/$backup"; break;} } -// 3. Last resort: an array-backed user share. +// 3. An array-backed user share. if (!$zip) foreach ($dir as $share) { if (!is_dir("/mnt/user/$share")) continue; $free = exec("df ".escapeshellarg("/mnt/user/$share")."|awk 'END{print $4}'"); if ($free > $used) {$zip = "/mnt/user/$share/$backup"; break;} } +// 4. Last resort: the boot device's own pool. /boot is always mounted (even with +// the array stopped), so this guarantees a target when nothing else is +// available. Only used for a real pool filesystem - a FAT/USB flash is not a +// pool and could not hold a >4GB archive anyway. +if (!$zip) { + $bootfs = exec("findmnt -no FSTYPE /boot"); + if (in_array($bootfs, ['zfs','btrfs','xfs'])) { + $free = exec("df /boot|awk 'END{print $4}'"); + if ($free > $used) $zip = "/boot/$backup"; + } +} if (!$zip) { write(_('Insufficient free disk space available')); @@ -94,11 +121,13 @@ if (!$zip) { exit(1); } -// Collect top-level entries to archive, skipping previous backups. +// Collect top-level entries to archive, skipping the prior-release/junk dirs +// above and any earlier flash-backup zip a user may have left on the flash. chdir('/boot'); $items = []; foreach (glob("*",GLOB_NOSORT) as $entry) { if (in_array($entry,$out)) continue; + if (preg_match('/-boot-backup-.*\.zip$/',$entry)) continue; $items[] = $entry; } $total = count($items); From f1dbf81db45ecae99a726b3e5b8565e2379e53e2 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Tue, 23 Jun 2026 11:14:34 -0400 Subject: [PATCH 06/16] feat(flash-backup): stream the archive directly to the browser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the staged background-job model with direct streaming: the zip is piped straight to the browser as it is built, so nothing is ever written to disk or RAM on the server. - scripts/flash_backup: now writes the zip to stdout (`zip -qr - ...`), usable both from the new endpoint and the CLI (`flash_backup 6 > out.zip`). No target selection, no nchan, no _() — just the exclusions and a single streaming zip. - include/FlashBackup.php: new download endpoint. Sets the attachment headers and `X-Accel-Buffering: no` (so nginx streams instead of spooling to a temp file), echoes the request's download token back as a cookie so the GUI can tell the stream started, then passes the script through. - BootInfo.page: clicking backup now navigates to the endpoint (browser handles the download, page isn't blocked) and shows a light "preparing…" status that clears as soon as the cookie appears. Removed the nchan/StartCommand path. Benefits: no staging file → no size/FAT limit, no cleanup/reaper, no target-selection logic, works with the array stopped, and it's faster (single pass; one zip invocation instead of per-item appends — ~44s vs ~115s on a test ZFS box). Trade-off: browser's native download indicator instead of an in-page progress log, and errors after streaming starts yield a truncated download. Verified on a live ZFS box: streamed archive validates, stderr clean (no zip pollution), previous/junk excluded, nothing left staged. Co-Authored-By: Claude Opus 4.8 --- emhttp/languages/en_US/helptext.txt | 4 +- emhttp/plugins/dynamix/BootInfo.page | 61 ++----- .../plugins/dynamix/include/FlashBackup.php | 51 ++++++ emhttp/plugins/dynamix/scripts/flash_backup | 155 +++--------------- 4 files changed, 93 insertions(+), 178 deletions(-) create mode 100644 emhttp/plugins/dynamix/include/FlashBackup.php diff --git a/emhttp/languages/en_US/helptext.txt b/emhttp/languages/en_US/helptext.txt index ba77faa2de..f21a0e350c 100644 --- a/emhttp/languages/en_US/helptext.txt +++ b/emhttp/languages/en_US/helptext.txt @@ -442,9 +442,9 @@ array or the cache disk/pool. :end :flash_backup_help: -Use *Boot device backup* to create a single zip file of the current contents of the boot device and store it locally on your computer. +Use *Boot device backup* to create a single zip file of the current contents of the boot device and download it to your computer. -The backup runs in the background and reports its progress, then downloads automatically when finished. Choose a *Compression level* before starting: *None* is the fastest but produces the largest file, while *Maximum* is the slowest but produces the smallest file. +The archive is streamed directly to your browser as it is built, so the download begins shortly after you start it and nothing is stored on the server. Choose a *Compression level* first: *None* is the fastest but produces the largest file, while *Maximum* is the slowest but produces the smallest file. The previous OS release and other non-essential files are excluded to keep the backup small. :end :syslinux_cfg_help: diff --git a/emhttp/plugins/dynamix/BootInfo.page b/emhttp/plugins/dynamix/BootInfo.page index 351158ac0d..2706e19c55 100644 --- a/emhttp/plugins/dynamix/BootInfo.page +++ b/emhttp/plugins/dynamix/BootInfo.page @@ -16,51 +16,26 @@ Tag="usb" ?> _(Active GUID)_: diff --git a/emhttp/plugins/dynamix/include/FlashBackup.php b/emhttp/plugins/dynamix/include/FlashBackup.php new file mode 100644 index 0000000000..be5e2bdd7d --- /dev/null +++ b/emhttp/plugins/dynamix/include/FlashBackup.php @@ -0,0 +1,51 @@ + + '/']); + +// Stream for as long as it takes, and stop zip if the user cancels the download. +set_time_limit(0); +ignore_user_abort(false); + +header('Content-Type: application/zip'); +header('Content-Disposition: attachment; filename="'.$name.'"'); +header('X-Accel-Buffering: no'); // tell nginx to stream, not spool to a temp file +header('Cache-Control: no-store'); +while (ob_get_level()) ob_end_clean(); + +passthru(escapeshellarg("$docroot/webGui/scripts/flash_backup")." ".(int)$level); +exit; +?> diff --git a/emhttp/plugins/dynamix/scripts/flash_backup b/emhttp/plugins/dynamix/scripts/flash_backup index 9485878683..022d84bb29 100755 --- a/emhttp/plugins/dynamix/scripts/flash_backup +++ b/emhttp/plugins/dynamix/scripts/flash_backup @@ -12,150 +12,39 @@ */ ?> backup.zip * compression-level 0 (store, fastest) .. 9 (smallest, slowest), default 6 - * nchan trailing marker added by StartCommand.php; ignored here * - * Launched as a tracked background job via StartCommand.php (same mechanism as - * plugin/docker operations), so it gets a PID, run de-duplication and abort for - * free. Progress is streamed to the 'flash_backup' nchan channel using the - * standard _DONE_/_ERROR_ sentinels; a _FILE_ message tells the GUI which - * archive to download. The work runs under nice + best-effort-low ionice and - * uses the streaming `zip` binary, so it stays gentle on CPU, disk and RAM - * even on large flash drives. The prior-release dirs (previous/prev) and - * desktop junk are excluded so the archive stays small. The finished zip is a - * throwaway staged in RAM (then pool/array, finally the boot device's own pool) - * and is deleted right after the browser downloads it. + * The archive is written straight to stdout so it can be streamed directly to a + * browser (see include/FlashBackup.php) without ever staging a file on disk or + * in RAM - this avoids any size limit, works with the array stopped, and needs + * no cleanup. zip runs under nice + best-effort-low ionice to stay gentle on + * CPU and disk. The prior-release dirs (previous/prev) and desktop junk are + * excluded so the archive stays small. */ -$docroot ??= ($_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp'); -require_once "$docroot/webGui/include/Wrappers.php"; -require_once "$docroot/webGui/include/publish.php"; -require_once "$docroot/webGui/include/Helpers.php"; // pools_filter() - -// This runs as a CLI background job (launched via StartCommand.php), where the -// GUI translation layer is not loaded. Provide a no-op _() fallback like other -// CLI scripts (e.g. monitor) so progress strings work; they stay in English. -if (!function_exists('_')) { function _($text) { return $text; } } - -$channel = 'flash_backup'; - -// Stream a progress line to the GUI over nchan. -function write(...$messages) { - global $channel; - foreach ($messages as $message) publish($channel, $message, 1, false); -} - -$var = (array)@parse_ini_file('/var/local/emhttp/var.ini'); -$disks = (array)@parse_ini_file('/var/local/emhttp/disks.ini', true); -$pools = pools_filter($disks); -$dir = ['system','appdata','isos','domains']; -// Top-level entries to exclude from the backup. 'previous'/'prev' hold the -// entire prior OS release (often >1GB) and must never bloat the archive; -// the rest is desktop junk created when the flash is mounted on a Mac/PC. -$out = ['prev','previous','System Volume Information','.Trash-0','.Trashes','.Spotlight-V100','.fseventsd','.TemporaryItems']; // Clamp the requested compression level to a valid zip range (default 6). -$level = isset($argv[1]) && is_numeric($argv[1]) ? (int)$argv[1] : 6; -$level = max(0, min(9, $level)); - -$server = isset($var['NAME']) ? str_replace(' ','_',strtolower($var['NAME'])) : 'tower'; -$mydate = date('Ymd-Hi'); -$osVersion = _var($var,'version','_unknown'); -$backup = "$server-v$osVersion-boot-backup-$mydate.zip"; - -// Safety net: reap any leftover staging archive from a previous run whose -// browser-side cleanup never fired (e.g. the tab was closed mid-download), so a -// stale copy can't linger - especially in RAM. The normal path still deletes -// the archive right after the download via Download.php cmd=unlink. -foreach (glob("$docroot/*-boot-backup-*.zip") as $stale) { - $target = @readlink($stale); - if ($target && is_file($target)) @unlink($target); - @unlink($stale); -} - -write(_('Preparing backup')."..."); +$level = isset($argv[1]) && is_numeric($argv[1]) ? max(0, min(9, (int)$argv[1])) : 6; -// Estimate required space (1.5x current flash usage) and pick a target, in -// order: the RAM-backed rootfs first, then a cache/SSD pool, then an array -// share. The archive is a throwaway staging file - built here, downloaded by -// the browser, then deleted - so RAM is the right home: it is always mounted -// (works even with the array stopped) and fastest. We only spill to disk when -// the flash is too large to stage in RAM with comfortable headroom. (Keeping -// the *process* RAM-light is handled separately by streaming via the zip -// binary rather than buffering the archive in memory.) -$used = exec("df /boot|awk 'END{print $3}'") * 1.5; -$zip = null; -// 1. RAM-backed rootfs (always available, fastest). -$free = exec("df /|awk 'END{print $4}'"); -if ($free > $used) $zip = "/$backup"; -// 2. Too big for RAM: a cache/SSD pool, written straight to the pool mount to -// bypass the slower /mnt/user FUSE layer. Pools require the array started. -if (!$zip) foreach ($pools as $pool) { - if (!is_dir("/mnt/$pool")) continue; - $free = exec("df ".escapeshellarg("/mnt/$pool")."|awk 'END{print $4}'"); - if ($free > $used) {$zip = "/mnt/$pool/$backup"; break;} -} -// 3. An array-backed user share. -if (!$zip) foreach ($dir as $share) { - if (!is_dir("/mnt/user/$share")) continue; - $free = exec("df ".escapeshellarg("/mnt/user/$share")."|awk 'END{print $4}'"); - if ($free > $used) {$zip = "/mnt/user/$share/$backup"; break;} -} -// 4. Last resort: the boot device's own pool. /boot is always mounted (even with -// the array stopped), so this guarantees a target when nothing else is -// available. Only used for a real pool filesystem - a FAT/USB flash is not a -// pool and could not hold a >4GB archive anyway. -if (!$zip) { - $bootfs = exec("findmnt -no FSTYPE /boot"); - if (in_array($bootfs, ['zfs','btrfs','xfs'])) { - $free = exec("df /boot|awk 'END{print $4}'"); - if ($free > $used) $zip = "/boot/$backup"; - } -} - -if (!$zip) { - write(_('Insufficient free disk space available')); - write('_ERROR_'); - exit(1); -} +// Top-level entries to exclude. 'previous'/'prev' hold the entire prior OS +// release (often >1GB); the rest is desktop junk created when the flash is +// mounted on a Mac/PC. +$out = ['prev','previous','System Volume Information','.Trash-0','.Trashes','.Spotlight-V100','.fseventsd','.TemporaryItems']; -// Collect top-level entries to archive, skipping the prior-release/junk dirs -// above and any earlier flash-backup zip a user may have left on the flash. chdir('/boot'); $items = []; -foreach (glob("*",GLOB_NOSORT) as $entry) { - if (in_array($entry,$out)) continue; - if (preg_match('/-boot-backup-.*\.zip$/',$entry)) continue; +foreach (glob('*', GLOB_NOSORT) as $entry) { + if (in_array($entry, $out)) continue; + if (preg_match('/-boot-backup-.*\.zip$/', $entry)) continue; // stray old backups $items[] = $entry; } -$total = count($items); - -// Run zip at low CPU/IO priority so the backup stays gentle without being -// starved: best-effort IO at the lowest priority (not the idle class, which can -// throttle throughput to a crawl on busy/ZFS systems). -$prio = 'nice -n 19 ionice -c2 -n7'; -$ok = true; -foreach ($items as $i => $entry) { - $pct = $total ? round(($i/$total)*100) : 0; - write(sprintf(_('Compressing %s')." (%d/%d - %d%%)", $entry, $i+1, $total, $pct)); - $flag = is_dir($entry) ? '-r' : ''; - exec("$prio zip -$level -q $flag ".escapeshellarg($zip)." ".escapeshellarg($entry)." 2>/dev/null", $o, $rc); - // zip rc 12 = "nothing to do"; treat any other non-zero result as a failure. - if ($rc !== 0 && $rc !== 12) $ok = false; -} - -if (!$ok || !is_file($zip)) { - @unlink($zip); - write(_('Backup failed')); - write('_ERROR_'); - exit(1); -} +if (!$items) exit(1); -// Expose the archive to the browser via a symlink in the webGui docroot. -symlink($zip,"$docroot/$backup"); -write(_('Backup complete')."..."); -write("_FILE_$backup"); -write('_DONE_'); +// Stream the archive to stdout. zip writes a valid streamable archive (data +// descriptors) when its output is not seekable. +$args = implode(' ', array_map('escapeshellarg', $items)); +passthru("nice -n 19 ionice -c2 -n7 zip -$level -qr - $args 2>/dev/null", $rc); +exit($rc); ?> From 71ae4cfd2c71cb000ae9326b6a940fec8655b563 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Tue, 23 Jun 2026 11:32:05 -0400 Subject: [PATCH 07/16] feat(flash-backup): real progress bar for the streamed download A streamed zip has no known final size, so the browser can't show a progress bar. Drive a real bar from the server instead, without giving up streaming: - scripts/flash_backup: add a `size` mode that prints the total byte size of the included set (single source of truth for the file list). - include/FlashBackup.php: pipe the zip to the client via popen and, as bytes flow, publish a percentage (bytes-sent / total) to the 'flash_backup' nchan channel, capped at 99% until the stream finishes (then _DONE_). The boot device is mostly already-compressed OS files, so output closely tracks input and the bar is accurate in practice. - BootInfo.page: subscribe to the channel and render a real progress bar that fills as the download streams, completing on _DONE_. Verified on a live ZFS box: size=2.55GB, bar climbs monotonically to ~96% then completes, streamed archive validates. Co-Authored-By: Claude Opus 4.8 --- emhttp/plugins/dynamix/BootInfo.page | 39 +++++++++------- .../plugins/dynamix/include/FlashBackup.php | 46 +++++++++++++------ emhttp/plugins/dynamix/scripts/flash_backup | 15 +++++- 3 files changed, 69 insertions(+), 31 deletions(-) diff --git a/emhttp/plugins/dynamix/BootInfo.page b/emhttp/plugins/dynamix/BootInfo.page index 2706e19c55..df5e9e6c64 100644 --- a/emhttp/plugins/dynamix/BootInfo.page +++ b/emhttp/plugins/dynamix/BootInfo.page @@ -17,25 +17,32 @@ Tag="usb" _(Active GUID)_: diff --git a/emhttp/plugins/dynamix/include/FlashBackup.php b/emhttp/plugins/dynamix/include/FlashBackup.php index be5e2bdd7d..d0df44f84c 100644 --- a/emhttp/plugins/dynamix/include/FlashBackup.php +++ b/emhttp/plugins/dynamix/include/FlashBackup.php @@ -12,16 +12,22 @@ ?> '/']); +// Total size of the included set, used as the progress denominator. +$total = (int)trim(shell_exec(escapeshellarg($script).' size 2>/dev/null')); -// Stream for as long as it takes, and stop zip if the user cancels the download. +// Stream for as long as it takes, and let zip die if the user cancels. set_time_limit(0); ignore_user_abort(false); @@ -46,6 +49,23 @@ header('Cache-Control: no-store'); while (ob_get_level()) ob_end_clean(); -passthru(escapeshellarg("$docroot/webGui/scripts/flash_backup")." ".(int)$level); +$h = popen(escapeshellarg($script).' '.(int)$level.' 2>/dev/null', 'r'); +if ($h) { + $sent = 0; $last = 0; + while (!feof($h)) { + $buf = fread($h, 1 << 18); + if ($buf === '' || $buf === false) break; + echo $buf; + flush(); + $sent += strlen($buf); + $now = microtime(true); + if ($total > 0 && $now - $last > 0.4) { + publish($channel, (string)min(99, intdiv($sent * 100, $total)), 1, false); + $last = $now; + } + } + pclose($h); +} +publish($channel, '_DONE_', 1, false); exit; ?> diff --git a/emhttp/plugins/dynamix/scripts/flash_backup b/emhttp/plugins/dynamix/scripts/flash_backup index 022d84bb29..441e20f94a 100755 --- a/emhttp/plugins/dynamix/scripts/flash_backup +++ b/emhttp/plugins/dynamix/scripts/flash_backup @@ -14,7 +14,8 @@ backup.zip + * Usage: flash_backup [compression-level] > backup.zip (stream the archive) + * flash_backup size (print total bytes) * compression-level 0 (store, fastest) .. 9 (smallest, slowest), default 6 * * The archive is written straight to stdout so it can be streamed directly to a @@ -23,8 +24,12 @@ * no cleanup. zip runs under nice + best-effort-low ionice to stay gentle on * CPU and disk. The prior-release dirs (previous/prev) and desktop junk are * excluded so the archive stays small. + * + * The 'size' mode prints the total byte size of the same included set; the + * endpoint uses it as the denominator for a download progress bar. */ +$sizeOnly = isset($argv[1]) && $argv[1] === 'size'; // Clamp the requested compression level to a valid zip range (default 6). $level = isset($argv[1]) && is_numeric($argv[1]) ? max(0, min(9, (int)$argv[1])) : 6; @@ -41,10 +46,16 @@ foreach (glob('*', GLOB_NOSORT) as $entry) { $items[] = $entry; } if (!$items) exit(1); +$args = implode(' ', array_map('escapeshellarg', $items)); + +// 'size' mode: print the total apparent byte size of the included set and exit. +if ($sizeOnly) { + echo (int)trim(exec("du -sbc $args 2>/dev/null | awk 'END{print $1}'")); + exit(0); +} // Stream the archive to stdout. zip writes a valid streamable archive (data // descriptors) when its output is not seekable. -$args = implode(' ', array_map('escapeshellarg', $items)); passthru("nice -n 19 ionice -c2 -n7 zip -$level -qr - $args 2>/dev/null", $rc); exit($rc); ?> From f53ce1d9e8781525b539f15ada5ea3d11409dbc0 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Tue, 23 Jun 2026 11:43:39 -0400 Subject: [PATCH 08/16] fix(flash-backup): download via hidden iframe so progress subscriber survives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The progress bar stayed at 0%: triggering the download with `window.location` starts a top-level navigation, and NchanSubscriber stops itself on the window's unload/beforeunload — so the subscriber died the instant the download began and no progress (nor _DONE_) ever reached the page. Trigger the download from a hidden iframe instead, so the main window never navigates and the subscriber keeps receiving progress. Also give the dialog a Close button as a safety and fix the now-inaccurate status text. Co-Authored-By: Claude Opus 4.8 --- emhttp/plugins/dynamix/BootInfo.page | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/emhttp/plugins/dynamix/BootInfo.page b/emhttp/plugins/dynamix/BootInfo.page index df5e9e6c64..746b254a4d 100644 --- a/emhttp/plugins/dynamix/BootInfo.page +++ b/emhttp/plugins/dynamix/BootInfo.page @@ -25,6 +25,7 @@ nchan_backup.on('message', function(data) { if (data == '_DONE_') { setBackupProgress(100); nchan_backup.stop(); + $('#fbFrame').remove(); setTimeout(function(){ swal.close(); }, 1200); return; } @@ -40,9 +41,12 @@ function setBackupProgress(pct) { function backup() { var level = $('#backupLevel').val(); nchan_backup.start(); - swal({title:"_(Boot device backup)_",text:"
0%
_(Your download will begin shortly)_
",html:true,animation:'none',showConfirmButton:false}); - // Attachment response keeps us on this page while the browser downloads. - window.location = '/webGui/include/FlashBackup.php?level='+level; + swal({title:"_(Boot device backup)_",text:"
0%
_(Backing up the boot device)_...
",html:true,animation:'none',showConfirmButton:true,confirmButtonText:"_(Close)_"}); + // Download via a hidden iframe (not window.location) so the page never starts + // to navigate: a top-level navigation fires 'beforeunload', which makes the + // NchanSubscriber stop itself - killing the progress updates. + $('#fbFrame').remove(); + $('