Sparse file support

cli/src/command/append.rs

@@ -446,6 +446,7 @@ fn append_to_archive(args: AppendCommand) -> anyhow::Result<()> {
             PathTransformers::new(args.substitutions, args.transforms),
             false,
         ),
+        sparse: false,
     };
 
     let archive = open_archive_then_seek_to_end(&archive_path)?;

cli/src/command/core.rs

@@ -20,9 +20,9 @@
 pub(crate) use self::timestamp::{TimeSource, TimestampStrategy};
 use crate::{
     cli::{CipherAlgorithmArgs, CompressionAlgorithmArgs, HashAlgorithmArgs, MissingTimePolicy},
-    utils::{self, PathPartExt, fs::HardlinkResolver},
+    utils::{self, PathPartExt, fs::HardlinkResolver, sparse::detect_sparse_map},
 };
 use anyhow::Context;
 pub(crate) use iter::ReorderByIndex;
 pub(crate) use path_filter::PathFilter;
 use path_slash::*;
@@ -42,7 +42,7 @@
 
 /// Detected format of an @archive source.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(crate) enum SourceFormat {
     /// PNA archive format (detected by magic bytes)
     Pna,
     /// mtree manifest format (text-based)
@@ -53,7 +53,7 @@
 ///
 /// Returns `SourceFormat::Pna` if the data starts with PNA magic bytes,
 /// otherwise returns `SourceFormat::Mtree`.
 pub(crate) fn detect_format<R: io::BufRead>(reader: &mut R) -> io::Result<SourceFormat> {
     let buf = reader.fill_buf()?;
 
     Ok(if buf.starts_with(PNA_HEADER) {
@@ -355,6 +355,7 @@
     pub(crate) option: WriteOptions,
     pub(crate) keep_options: KeepOptions,
     pub(crate) pathname_editor: PathnameEditor,
+    pub(crate) sparse: bool,
 }
 
 #[derive(Clone, Debug)]
@@ -810,6 +811,41 @@
     Ok(())
 }
 
+/// Writes file data from a path, detecting and preserving sparse regions.
+///
+/// If the file is sparse, only data regions are written and the sparse map is set on the entry.
+/// If the file is not sparse, falls back to normal write behavior.
+pub(crate) fn write_sparse_from_path(
+    entry: &mut EntryBuilder,
+    path: impl AsRef<Path>,
+) -> io::Result<()> {
+    use io::Seek;
+
+    let path = path.as_ref();
+    let mut file = fs::File::open(path)?;
+
+    if let Some(sparse_map) = detect_sparse_map(&file)? {
+        // Write only data regions using chunked I/O to avoid memory exhaustion
+        const CHUNK_SIZE: usize = 64 * 1024;
+        let mut buf = vec![0u8; CHUNK_SIZE];
+        for region in sparse_map.regions() {
+            file.seek(io::SeekFrom::Start(region.offset()))?;
+            let mut remaining = region.size();
+            while remaining > 0 {
+                let to_read = (remaining as usize).min(CHUNK_SIZE);
+                file.read_exact(&mut buf[..to_read])?;
+                entry.write_all(&buf[..to_read])?;
+                remaining -= to_read as u64;
+            }
+        }
+        entry.set_sparse_map(sparse_map);
+        Ok(())
+    } else {
+        // Not sparse, use normal write
+        write_from_path(entry, path)
+    }
+}
+
 #[inline]
 pub(crate) fn write_from_path(writer: &mut impl Write, path: impl AsRef<Path>) -> io::Result<()> {
     let path = path.as_ref();
@@ -843,6 +879,7 @@
         option,
         keep_options,
         pathname_editor,
+        sparse,
     }: &CreateOptions,
 ) -> io::Result<Option<NormalEntry>> {
     let CollectedEntry {
@@ -869,7 +906,11 @@
         }
         StoreAs::File => {
             let mut entry = EntryBuilder::new_file(entry_name, option)?;
-            write_from_path(&mut entry, path)?;
+            if *sparse {
+                write_sparse_from_path(&mut entry, path)?;
+            } else {
+                write_from_path(&mut entry, path)?;
+            }
             apply_metadata(entry, path, keep_options, metadata)?.build()
         }
         StoreAs::Dir => {

cli/src/command/core/mtree.rs

@@ -92,6 +92,7 @@ fn create_entry_from_mtree(
         option,
         keep_options,
         pathname_editor,
+        sparse: _,
     }: &CreateOptions,
 ) -> io::Result<Option<NormalEntry>> {
     let entry_path = mtree_entry.path();

cli/src/command/create.rs

@@ -46,6 +46,7 @@ use std::{
     group(ArgGroup::new("user-flag").args(["numeric_owner", "uname"])),
     group(ArgGroup::new("group-flag").args(["numeric_owner", "gname"])),
     group(ArgGroup::new("recursive-flag").args(["recursive", "no_recursive"])),
+    group(ArgGroup::new("sparse-flag").args(["sparse", "no_sparse"])),
     group(ArgGroup::new("keep-dir-flag").args(["keep_dir", "no_keep_dir"])),
     group(ArgGroup::new("keep-xattr-flag").args(["keep_xattr", "no_keep_xattr"])),
     group(ArgGroup::new("keep-timestamp-flag").args(["keep_timestamp", "no_keep_timestamp"])),
@@ -164,6 +165,18 @@ pub(crate) struct CreateCommand {
         help = "Compress multiple files together for better compression ratio"
     )]
     solid: bool,
+    #[arg(
+        long,
+        requires = "unstable",
+        help = "Detect and preserve sparse files (unstable)"
+    )]
+    sparse: bool,
+    #[arg(
+        long,
+        requires = "unstable",
+        help = "Do not detect sparse files. This is the inverse option of --sparse (unstable)"
+    )]
+    no_sparse: bool,
     #[arg(long, value_name = "NAME", help = "Set user name for archive entries")]
     uname: Option<String>,
     #[arg(long, value_name = "NAME", help = "Set group name for archive entries")]
@@ -533,6 +546,7 @@ fn create_archive(args: CreateCommand) -> anyhow::Result<()> {
         write_option,
         keep_options,
         solid: args.solid,
+        sparse: args.sparse,
         pathname_editor,
     };
     if let Some(size) = max_file_size {
@@ -568,6 +582,7 @@ pub(crate) struct CreationContext {
     pub(crate) write_option: WriteOptions,
     pub(crate) keep_options: KeepOptions,
     pub(crate) solid: bool,
+    pub(crate) sparse: bool,
     pub(crate) pathname_editor: PathnameEditor,
 }
 
@@ -577,6 +592,7 @@ pub(crate) fn create_archive_file<W, F>(
         write_option,
         keep_options,
         solid,
+        sparse,
         pathname_editor,
     }: CreationContext,
     target_items: Vec<CollectedItem>,
@@ -598,6 +614,7 @@ where
         option,
         keep_options,
         pathname_editor,
+        sparse,
     };
     let rx = spawn_entry_results(
         target_items,
@@ -638,6 +655,7 @@ fn create_archive_with_split(
         write_option,
         keep_options,
         solid,
+        sparse,
         pathname_editor,
     }: CreationContext,
     target_items: Vec<CollectedItem>,
@@ -656,6 +674,7 @@ fn create_archive_with_split(
         option,
         keep_options,
         pathname_editor,
+        sparse,
     };
     let rx = spawn_entry_results(
         target_items,

cli/src/command/extract.rs

@@ -26,7 +26,10 @@
 };
 use anyhow::Context;
 use clap::{ArgGroup, Parser, ValueHint};
-use pna::{DataKind, EntryName, EntryReference, NormalEntry, Permission, ReadOptions, prelude::*};
+use pna::{
+    DataKind, EntryName, EntryReference, NormalEntry, Permission, ReadOptions, SparseMap,
+    prelude::*,
+};
 #[cfg(target_os = "macos")]
 use std::os::macos::fs::FileTimesExt;
 #[cfg(windows)]
@@ -899,7 +902,9 @@
 
     match entry_kind {
         DataKind::File => {
-            if *safe_writes {
+            let sparse_map = item.sparse_map();
+            if *safe_writes && sparse_map.is_none() {
+                // Safe writes (atomic rename) - only for non-sparse files
                 let mut safe_writer = SafeWriter::new(&path)?;
                 {
                     let mut writer =
@@ -911,6 +916,12 @@
                 // Set timestamps before persist; after rename we lose the file handle
                 restore_timestamps(safe_writer.as_file_mut(), item.metadata(), keep_options)?;
                 safe_writer.persist()?;
+            } else if let Some(sparse_map) = sparse_map {
+                // Sparse file restoration - write data regions at correct offsets
+                let mut file = utils::fs::file_create(&path, remove_existing)?;
+                let mut reader = item.reader(ReadOptions::with_password(password))?;
+                restore_sparse_file(&mut file, &mut reader, sparse_map)?;
+                restore_timestamps(&mut file, item.metadata(), keep_options)?;
             } else {
                 if remove_existing {
                     utils::io::ignore_not_found(utils::fs::remove_path(&path))?;
@@ -1057,7 +1068,7 @@
         MacMetadataStrategy::Always
     ) && item.mac_metadata().is_some();
     #[cfg(not(target_os = "macos"))]
     let skip_xattr_acl = false;
 
     #[cfg(unix)]
     if !skip_xattr_acl {
@@ -1311,6 +1322,65 @@
     Ok(())
 }
 
+/// Restores a sparse file by writing only data regions and seeking over holes.
+///
+/// This creates a sparse file on filesystems that support it by using seek
+/// to skip over hole regions instead of writing zeros.
+fn restore_sparse_file(
+    file: &mut fs::File,
+    reader: &mut impl Read,
+    sparse_map: &SparseMap,
+) -> io::Result<()> {
+    use io::Seek;
+
+    let expected_data_size = sparse_map.data_size().ok_or_else(|| {
+        io::Error::new(
+            io::ErrorKind::InvalidData,
+            "Sparse map data size overflow (corrupted archive)",
+        )
+    })?;
+
+    // Write each data region at its correct offset using chunked I/O
+    const CHUNK_SIZE: usize = 64 * 1024;
+    let mut buf = vec![0u8; CHUNK_SIZE];
+    let mut total_read = 0u64;
+    for region in sparse_map.regions() {
+        file.seek(io::SeekFrom::Start(region.offset()))?;
+        let mut remaining = region.size();
+        while remaining > 0 {
+            let to_read = (remaining as usize).min(CHUNK_SIZE);
+            reader.read_exact(&mut buf[..to_read]).map_err(|e| {
+                io::Error::new(
+                    e.kind(),
+                    format!(
+                        "Failed to read sparse data at offset {}: {} \
+                         (expected {} bytes total, read {} so far)",
+                        region.offset(),
+                        e,
+                        expected_data_size,
+                        total_read
+                    ),
+                )
+            })?;
+            file.write_all(&buf[..to_read])?;
+            remaining -= to_read as u64;
+            total_read += to_read as u64;
+        }
+    }
+
+    // Verify total bytes read matches expected
+    debug_assert_eq!(
+        total_read, expected_data_size,
+        "Sparse data size mismatch: read {}, expected {}",
+        total_read, expected_data_size
+    );
+
+    // Set the file length to the logical size (handles trailing holes)
+    file.set_len(sparse_map.logical_size())?;
+
+    Ok(())
+}
+
 fn ensure_directory_components(path: &Path, unlink_first: bool) -> io::Result<()> {
     if path.as_os_str().is_empty() {
         return Ok(());

cli/src/command/stdio.rs

@@ -951,6 +951,7 @@ fn run_create_archive(args: StdioCommand) -> anyhow::Result<()> {
         write_option: cli_option,
         keep_options,
         solid: args.solid,
+        sparse: false,
         pathname_editor: PathnameEditor::new(
             args.strip_components,
             PathTransformers::new(args.substitutions, args.transforms),
@@ -1258,6 +1259,7 @@ fn run_append(args: StdioCommand) -> anyhow::Result<()> {
             PathTransformers::new(args.substitutions, args.transforms),
             args.absolute_paths,
         ),
+        sparse: false,
     };
 
     // NOTE: "-" will use stdin/out
@@ -1412,6 +1414,7 @@ fn run_update(args: StdioCommand) -> anyhow::Result<()> {
             PathTransformers::new(args.substitutions, args.transforms),
             args.absolute_paths,
         ),
+        sparse: false,
     };
 
     // NOTE: "-" is not supported for update mode

cli/src/command/update.rs

@@ -450,6 +450,7 @@ fn update_archive(args: UpdateCommand) -> anyhow::Result<()> {
             PathTransformers::new(args.substitutions, args.transforms),
             false,
         ),
+        sparse: false,
     };
 
     let archives = collect_split_archives(&args.file.archive)?;

cli/src/utils.rs

@@ -10,6 +10,7 @@ pub(crate) mod mmap;
 pub(crate) mod os;
 mod path;
 pub(crate) mod process;
+pub(crate) mod sparse;
 pub(crate) mod str;
 
 pub(crate) use {globs::*, path::*};