feat: reduce pack buffer pool memory ~40% (640→384 MiB)

Lower default upload concurrency from 4 to 2 (pool 5→3 buffers) and
shrink oversized buffers on pool return to reclaim memory from packs
that grew beyond target size. Consolidate the default into a single
pub const in config::limits.

Author: m3nu

crates/vger-core/src/config/limits.rs

@@ -2,6 +2,9 @@ use serde::{Deserialize, Serialize};
 
 use crate::error::{Result, VgerError};
 
+/// Default maximum number of in-flight background pack uploads.
+pub const DEFAULT_UPLOAD_CONCURRENCY: usize = 2;
+
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct ResourceLimitsConfig {
     #[serde(default)]
@@ -26,7 +29,7 @@ pub struct CpuLimitsConfig {
     /// Unix process niceness target (-20..19). 0 = unchanged.
     #[serde(default)]
     pub nice: i32,
-    /// Max in-flight background pack uploads (default: 4, range: 1-16).
+    /// Max in-flight background pack uploads (default: 2, range: 1-16).
     #[serde(default)]
     pub max_upload_concurrency: Option<usize>,
     /// Batch size in MiB for transform flushes (default: 32, range: 4-256).
@@ -101,7 +104,8 @@ impl CpuLimitsConfig {
 
     /// Effective upload concurrency limit.
     pub fn upload_concurrency(&self) -> usize {
-        self.max_upload_concurrency.unwrap_or(4)
+        self.max_upload_concurrency
+            .unwrap_or(DEFAULT_UPLOAD_CONCURRENCY)
     }
 
     /// Effective pipeline depth (0 = disabled).

crates/vger-core/src/config/mod.rs

@@ -12,6 +12,7 @@ pub(crate) use self::hooks::HOOK_COMMANDS;
 pub use self::hooks::{HooksConfig, SourceHooksConfig};
 pub use self::limits::{
     CpuLimitsConfig, IoLimitsConfig, NetworkLimitsConfig, ResourceLimitsConfig,
+    DEFAULT_UPLOAD_CONCURRENCY,
 };
 
 // Re-export from submodules

crates/vger-core/src/repo/mod.rs

@@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize};
 use tracing::{debug, warn};
 
 use crate::compress;
-use crate::config::{ChunkerConfig, RepositoryConfig};
+use crate::config::{ChunkerConfig, RepositoryConfig, DEFAULT_UPLOAD_CONCURRENCY};
 use crate::crypto::chunk_id::ChunkId;
 use crate::crypto::key::{EncryptedKey, MasterKey};
 use crate::crypto::pack_id::PackId;
@@ -55,9 +55,6 @@ fn default_max_pack_size() -> u32 {
     128 * 1024 * 1024
 }
 
-/// Default maximum number of in-flight background pack uploads.
-const DEFAULT_IN_FLIGHT_PACK_UPLOADS: usize = 4;
-
 /// Maximum total weight (bytes) of cached blobs in the blob cache.
 const BLOB_CACHE_MAX_BYTES: usize = 32 * 1024 * 1024; // 32 MiB
 
@@ -286,7 +283,7 @@ impl Repository {
             index_delta: None,
             blob_cache: BlobCache::new(BLOB_CACHE_MAX_BYTES),
             pack_buffer_pool: None,
-            max_in_flight_uploads: DEFAULT_IN_FLIGHT_PACK_UPLOADS,
+            max_in_flight_uploads: DEFAULT_UPLOAD_CONCURRENCY,
             manifest_dirty: false,
             index_dirty: false,
             file_cache_dirty: false,
@@ -406,7 +403,7 @@ impl Repository {
             index_delta: None,
             blob_cache: BlobCache::new(BLOB_CACHE_MAX_BYTES),
             pack_buffer_pool: None,
-            max_in_flight_uploads: DEFAULT_IN_FLIGHT_PACK_UPLOADS,
+            max_in_flight_uploads: DEFAULT_UPLOAD_CONCURRENCY,
             manifest_dirty: false,
             index_dirty: false,
             file_cache_dirty: false,
@@ -554,7 +551,8 @@ impl Repository {
     /// the rest for in-flight uploads. Buffers start empty and grow lazily.
     pub fn activate_pack_buffer_pool(&mut self) {
         let pool_size = self.max_in_flight_uploads + 1;
-        self.pack_buffer_pool = Some(Arc::new(PackBufferPool::new(pool_size)));
+        let target = self.data_pack_writer.target_size();
+        self.pack_buffer_pool = Some(Arc::new(PackBufferPool::new(pool_size, target)));
     }
 
     #[cfg(test)]

crates/vger-core/src/repo/pack.rs

@@ -196,6 +196,11 @@ impl PackWriter {
         self.pack_type
     }
 
+    /// The target pack size in bytes.
+    pub fn target_size(&self) -> usize {
+        self.target_size
+    }
+
     /// Append the encrypted header trailer to the already-accumulated `pack_bytes`,
     /// compute the PackId, and swap the finalized buffer into `out`.
     ///
@@ -421,18 +426,24 @@ pub fn compute_tree_pack_target(min_pack_size: u32) -> usize {
 pub struct PackBufferPool {
     inner: Mutex<VecDeque<Vec<u8>>>,
     available: Condvar,
+    /// Desired buffer capacity. Buffers that grow beyond 125% of this are
+    /// shrunk on return to reclaim memory from oversized final packs.
+    target_capacity: usize,
 }
 
 impl PackBufferPool {
     /// Create a pool with `count` empty buffers.
-    pub fn new(count: usize) -> Self {
+    /// `target_capacity` is the expected buffer size; buffers that grow beyond
+    /// 125% of this value are shrunk on return to reclaim memory.
+    pub fn new(count: usize, target_capacity: usize) -> Self {
         let mut bufs = VecDeque::with_capacity(count);
         for _ in 0..count {
             bufs.push_back(Vec::new());
         }
         Self {
             inner: Mutex::new(bufs),
             available: Condvar::new(),
+            target_capacity,
         }
     }
 
@@ -452,7 +463,11 @@ impl PackBufferPool {
     }
 
     /// Return a buffer to the pool. Private — only called by `PooledBuffer::drop`.
-    fn checkin(&self, buf: Vec<u8>) {
+    fn checkin(&self, mut buf: Vec<u8>) {
+        // Reclaim memory from buffers that grew beyond target (e.g. large final blob).
+        if buf.capacity() > self.target_capacity + self.target_capacity / 4 {
+            buf.shrink_to(self.target_capacity);
+        }
         let mut guard = self.inner.lock().unwrap();
         guard.push_back(buf);
         self.available.notify_one();
@@ -556,7 +571,7 @@ mod tests {
 
     #[test]
     fn pool_checkout_and_return() {
-        let pool = Arc::new(PackBufferPool::new(2));
+        let pool = Arc::new(PackBufferPool::new(2, 128 * 1024));
 
         // Can check out up to pool size
         let buf1 = pool.checkout();
@@ -576,7 +591,7 @@ mod tests {
 
     #[test]
     fn pool_blocks_when_exhausted() {
-        let pool = Arc::new(PackBufferPool::new(1));
+        let pool = Arc::new(PackBufferPool::new(1, 128 * 1024));
         let _buf1 = pool.checkout();
 
         // Spawn a thread that tries to checkout — it should block
@@ -598,7 +613,7 @@ mod tests {
 
     #[test]
     fn pool_retains_buffer_capacity() {
-        let pool = Arc::new(PackBufferPool::new(1));
+        let pool = Arc::new(PackBufferPool::new(1, 128 * 1024));
 
         // Check out, grow the buffer, return it
         {