feat: range request support for getBytes and getStream

drivers/fs/driver.ts

@@ -12,6 +12,7 @@ import mimeTypes from 'mime-types'
 import * as fsp from 'node:fs/promises'
 import { fileURLToPath } from 'node:url'
 import { type Readable } from 'node:stream'
+import { buffer } from 'node:stream/consumers'
 import string from '@poppinss/utils/string'
 import { Retrier } from '@humanwhocodes/retry'
 import { dirname, join, relative } from 'node:path'
@@ -22,7 +23,13 @@ import debug from './debug.js'
 import type { FSDriverOptions } from './types.js'
 import { DriveFile } from '../../src/driver_file.js'
 import { DriveDirectory } from '../../src/drive_directory.js'
+import {
+  isRangeRequest,
+  validateRangeRequest,
+  validateRangeSatisfiable,
+} from '../../src/range_utils.js'
 import type {
+  ReadOptions,
   WriteOptions,
   ObjectMetaData,
   DriverContract,
@@ -102,6 +109,19 @@ export class FSDriver implements DriverContract {
     })
   }
 
+  /**
+   * Creates a readable stream for the given key, optionally restricted to a byte range.
+   */
+  async #createReadStream(key: string, options?: ReadOptions): Promise<Readable> {
+    const location = join(this.#rootUrl, key)
+    if (isRangeRequest(options?.range)) {
+      validateRangeRequest(key, options.range)
+      const { size } = await fsp.stat(location)
+      validateRangeSatisfiable(key, options.range, size)
+    }
+    return createReadStream(location, options?.range)
+  }
+
   /**
    * Synchronously check if a file exists
    */
@@ -140,19 +160,23 @@ export class FSDriver implements DriverContract {
   /**
    * Returns the contents of the file as a stream. An
    * exception is thrown when the file is missing.
+   * When a range is provided, only the specified bytes are streamed.
    */
-  async getStream(key: string): Promise<Readable> {
+  async getStream(key: string, options?: ReadOptions): Promise<Readable> {
     debug('reading file contents as a stream %s:%s', this.#rootUrl, key)
-    const location = join(this.#rootUrl, key)
-    return createReadStream(location)
+    return this.#createReadStream(key, options)
   }
 
   /**
    * Returns the contents of the file as an Uint8Array. An
    * exception is thrown when the file is missing.
+   * When a range is provided, only the specified bytes are returned.
    */
-  async getBytes(key: string): Promise<Uint8Array> {
+  async getBytes(key: string, options?: ReadOptions): Promise<Uint8Array> {
     debug('reading file contents as array buffer %s:%s', this.#rootUrl, key)
+    if (isRangeRequest(options?.range)) {
+      return new Uint8Array(await buffer(await this.#createReadStream(key, options)))
+    }
     return this.#read(key).then((value) => new Uint8Array(value.buffer))
   }
 

drivers/gcs/driver.ts

@@ -8,6 +8,7 @@
  */
 
 import { type Readable } from 'node:stream'
+import { buffer } from 'node:stream/consumers'
 import string from '@poppinss/utils/string'
 import {
   Storage,
@@ -21,7 +22,13 @@ import debug from './debug.js'
 import type { GCSDriverOptions } from './types.js'
 import { DriveFile } from '../../src/driver_file.js'
 import { DriveDirectory } from '../../src/drive_directory.js'
+import {
+  isRangeRequest,
+  validateRangeRequest,
+  validateRangeSatisfiable,
+} from '../../src/range_utils.js'
 import type {
+  ReadOptions,
   WriteOptions,
   CopyMoveOptions,
   ObjectMetaData,
@@ -153,6 +160,20 @@ export class GCSDriver implements DriverContract {
     })
   }
 
+  /**
+   * Creates a readable stream for the given key, optionally restricted to a byte range.
+   */
+  async #createReadStream(key: string, options?: ReadOptions): Promise<Readable> {
+    const file = this.#storage.bucket(this.options.bucket).file(key)
+    if (isRangeRequest(options?.range)) {
+      validateRangeRequest(key, options.range)
+      const [metadata] = await file.getMetadata()
+      const size = Number(metadata.size)
+      validateRangeSatisfiable(key, options.range, size)
+    }
+    return file.createReadStream(options?.range)
+  }
+
   /**
    * Returns a boolean indicating if the file exists
    * or not.
@@ -180,22 +201,24 @@ export class GCSDriver implements DriverContract {
   /**
    * Returns the contents of the file as a Readable stream. An
    * exception is thrown when the file is missing.
+   * When a range is provided, only the specified bytes are streamed.
    */
-  async getStream(key: string): Promise<Readable> {
+  async getStream(key: string, options?: ReadOptions): Promise<Readable> {
     debug('reading file contents as a stream %s:%s', this.options.bucket, key)
-    const bucket = this.#storage.bucket(this.options.bucket)
-
-    return bucket.file(key).createReadStream()
+    return this.#createReadStream(key, options)
   }
 
   /**
    * Returns the contents of the file as an Uint8Array. An
    * exception is thrown when the file is missing.
+   * When a range is provided, only the specified bytes are returned.
    */
-  async getBytes(key: string): Promise<Uint8Array> {
+  async getBytes(key: string, options?: ReadOptions): Promise<Uint8Array> {
     debug('reading file contents as array buffer %s:%s', this.options.bucket, key)
+    if (isRangeRequest(options?.range)) {
+      return new Uint8Array(await buffer(await this.#createReadStream(key, options)))
+    }
     const bucket = this.#storage.bucket(this.options.bucket)
-
     const response = await bucket.file(key).download()
     return new Uint8Array(response[0])
   }

drivers/s3/driver.ts

@@ -37,7 +37,13 @@ import debug from './debug.js'
 import { type S3DriverOptions } from './types.js'
 import { DriveFile } from '../../src/driver_file.js'
 import { DriveDirectory } from '../../src/drive_directory.js'
+import {
+  isRangeRequest,
+  validateRangeRequest,
+  validateRangeSatisfiable,
+} from '../../src/range_utils.js'
 import type {
+  ReadOptions,
   WriteOptions,
   CopyMoveOptions,
   DriverContract,
@@ -314,35 +320,47 @@ export class S3Driver implements DriverContract {
   }
 
   /**
-   * Returns the contents of the file as a Readable stream. An
-   * exception is thrown when the file is missing.
+   * Sends a GetObjectCommand with optional range request handling
    */
-  async getStream(key: string): Promise<Readable> {
-    debug('reading file contents as a stream %s:%s', this.options.bucket, key)
-    const response = await this.#client.send(
+  async #getObject(key: string, options?: ReadOptions) {
+    let rangeHeader = {}
+    if (isRangeRequest(options?.range)) {
+      validateRangeRequest(key, options.range)
+      const head = await this.#client.send(
+        this.createHeadObjectCommand(this.#client, { Key: key, Bucket: this.options.bucket })
+      )
+      validateRangeSatisfiable(key, options.range, head.ContentLength!)
+      rangeHeader = { Range: `bytes=${options.range.start ?? 0}-${options.range.end ?? ''}` }
+    }
+    return this.#client.send(
       this.createGetObjectCommand(this.#client, {
         Key: key,
         Bucket: this.options.bucket,
+        ...rangeHeader,
       })
     )
+  }
 
-    return response.Body! as Readable
+  /**
+   * Returns the contents of the file as a Readable stream. An
+   * exception is thrown when the file is missing.
+   * When a range is provided, only the specified bytes are streamed.
+   */
+  async getStream(key: string, options?: ReadOptions): Promise<Readable> {
+    debug('reading file contents as a stream %s:%s', this.options.bucket, key)
+    const { Body } = await this.#getObject(key, options)
+    return Body! as Readable
   }
 
   /**
    * Returns the contents of the file as an Uint8Array. An
    * exception is thrown when the file is missing.
+   * When a range is provided, only the specified bytes are returned.
    */
-  async getBytes(key: string): Promise<Uint8Array> {
+  async getBytes(key: string, options?: ReadOptions): Promise<Uint8Array> {
     debug('reading file contents as array buffer %s:%s', this.options.bucket, key)
-    const response = await this.#client.send(
-      this.createGetObjectCommand(this.#client, {
-        Key: key,
-        Bucket: this.options.bucket,
-      })
-    )
-
-    return response.Body!.transformToByteArray()
+    const { Body } = await this.#getObject(key, options)
+    return Body!.transformToByteArray()
   }
 
   /**

src/disk.ts

@@ -16,6 +16,7 @@ import { DriveFile } from './driver_file.js'
 import { KeyNormalizer } from './key_normalizer.js'
 import { type DriveDirectory } from './drive_directory.js'
 import type {
+  ReadOptions,
   WriteOptions,
   FileSnapshot,
   ObjectMetaData,
@@ -76,15 +77,15 @@ export class Disk {
   /**
    * Returns file contents as a Readable stream.
    */
-  getStream(key: string): Promise<Readable> {
-    return this.file(key).getStream()
+  getStream(key: string, options?: ReadOptions): Promise<Readable> {
+    return this.file(key).getStream(options)
   }
 
   /**
    * Returns file contents as a Uint8Array.
    */
-  getBytes(key: string): Promise<Uint8Array> {
-    return this.file(key).getBytes()
+  getBytes(key: string, options?: ReadOptions): Promise<Uint8Array> {
+    return this.file(key).getBytes(options)
   }
 
   /**

src/driver_file.ts

@@ -13,6 +13,7 @@ import { type Readable } from 'node:stream'
 import * as errors from './errors.js'
 import { KeyNormalizer } from './key_normalizer.js'
 import type {
+  ReadOptions,
   DriverContract,
   FileSnapshot,
   ObjectMetaData,
@@ -92,9 +93,9 @@ export class DriveFile {
   /**
    * Returns file contents as a Readable stream.
    */
-  async getStream(): Promise<Readable> {
+  async getStream(options?: ReadOptions): Promise<Readable> {
     try {
-      return await this.#driver.getStream(this.key)
+      return await this.#driver.getStream(this.key, options)
     } catch (error) {
       throw new errors.E_CANNOT_READ_FILE([this.key], { cause: error })
     }
@@ -103,9 +104,9 @@ export class DriveFile {
   /**
    * Returns file contents as a Uint8Array.
    */
-  async getBytes(): Promise<Uint8Array> {
+  async getBytes(options?: ReadOptions): Promise<Uint8Array> {
     try {
-      return await this.#driver.getBytes(this.key)
+      return await this.#driver.getBytes(this.key, options)
     } catch (error) {
       throw new errors.E_CANNOT_READ_FILE([this.key], { cause: error })
     }

src/errors.ts

@@ -112,3 +112,11 @@ export const E_PATH_TRAVERSAL_DETECTED = createError<[key: string]>(
   'Path traversal segment detected in key "%s"',
   'E_PATH_TRAVERSAL_DETECTED'
 )
+
+/**
+ * The requested byte range is invalid or not satisfiable
+ */
+export const E_RANGE_UNSATISFIABLE = createError<[key: string]>(
+  'The specified range is invalid or exceeds the file size for "%s"',
+  'E_RANGE_UNSATISFIABLE'
+)

src/range_utils.ts

@@ -0,0 +1,59 @@
+/*
+ * flydrive
+ *
+ * (c) FlyDrive
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+import * as errors from './errors.js'
+import type { RangeRequest } from './types.js'
+
+/**
+ * Returns true if the range has at least one bound defined.
+ * Guards against empty range objects being treated as ranged reads.
+ */
+export function isRangeRequest(range?: RangeRequest): range is RangeRequest {
+  return range !== undefined && (range.start !== undefined || range.end !== undefined)
+}
+
+/**
+ * Validates the syntax of a range request. Checks for negative values,
+ * non-integers, and start > end. Should be called before any I/O.
+ */
+export function validateRangeRequest(key: string, range: RangeRequest): void {
+  const { start, end } = range
+
+  if (start !== undefined && (!Number.isInteger(start) || start < 0)) {
+    throw new errors.E_RANGE_UNSATISFIABLE([key])
+  }
+
+  if (end !== undefined && (!Number.isInteger(end) || end < 0)) {
+    throw new errors.E_RANGE_UNSATISFIABLE([key])
+  }
+
+  if (start !== undefined && end !== undefined && start > end) {
+    throw new errors.E_RANGE_UNSATISFIABLE([key])
+  }
+}
+
+/**
+ * Validates that the range falls within the known content length.
+ * Should be called after a preflight stat or metadata fetch.
+ */
+export function validateRangeSatisfiable(
+  key: string,
+  range: RangeRequest,
+  contentLength: number
+): void {
+  const { start, end } = range
+
+  if (start !== undefined && start >= contentLength) {
+    throw new errors.E_RANGE_UNSATISFIABLE([key])
+  }
+
+  if (end !== undefined && end >= contentLength) {
+    throw new errors.E_RANGE_UNSATISFIABLE([key])
+  }
+}

src/types.ts

@@ -72,6 +72,24 @@ export type UploadSignedURLOptions = {
   [key: string]: any
 }
 
+/**
+ * Represents an inclusive byte range for partial reads.
+ * Both start and end must be positive integers representing
+ * absolute byte positions. Both start and end are inclusive
+ * and start counting at 0.
+ */
+export type RangeRequest = {
+  start?: number
+  end?: number
+}
+
+/**
+ * Options accepted by read operations.
+ */
+export type ReadOptions = {
+  range?: RangeRequest
+}
+
 /**
  * Representation of file snapshot. It can be persisted
  * inside any database storage.
@@ -106,14 +124,14 @@ export interface DriverContract {
    * Should throw "E_CANNOT_READ_FILE" error when the file
    * does not exists.
    */
-  getStream(key: string): Promise<Readable>
+  getStream(key: string, options?: ReadOptions): Promise<Readable>
 
   /**
    * Return contents of an object for the given key as an Uint8Array.
    * Should throw "E_CANNOT_READ_FILE" error when the file
    * does not exists.
    */
-  getBytes(key: string): Promise<Uint8Array>
+  getBytes(key: string, options?: ReadOptions): Promise<Uint8Array>
 
   /**
    * Return metadata of an object for the given key.