feat(uploader): add chunked file upload

README.md

@@ -90,6 +90,43 @@ Easy to use Angular2 directives for files upload ([demo](http://valor-software.g
   [html demo](https://github.com/valor-software/ng2-file-upload/blob/master/demo/components/file-upload/simple-demo.html)
   - `onFileDrop` - it fires after a file has been dropped on a Drop Area; you can pass in `$event` to get the list of files that were dropped. i.e. `(onFileDrop)="dropped($event)"`
 
+## Chunked uploads
+
+  Use `ChunkedFileUploader` (a `FileUploader` subclass) instead of `FileUploader` to send large files in several requests. It works with the same directives, options and callbacks; without `chunkSize` it behaves exactly like `FileUploader`.
+
+  ```typescript
+  uploader = new ChunkedFileUploader({ url: URL, chunkSize: 2 * 1024 * 1024 });
+  ```
+
+  Additional options:
+
+  1. `chunkSize` - Bytes per request. Multipart requests send the chunk as the file field, preceded by `chunkIndex` and `totalChunks` fields; with `disableMultipart` the raw chunk is sent with a `Content-Range` header (`formatDataFunction` is not used).
+  2. `chunkIndexParam` / `totalChunksParam` - Names of those form fields. Default to `chunkIndex` and `totalChunks`.
+
+  Additional callbacks and methods:
+
+  - `onBeforeUploadChunk(item, chunk)` - before each chunk request.
+  - `onSuccessChunk(item, chunk, response, status, headers)` - after each successful chunk. Change `item.url`, `item.method` or `item.headers` here to target the next chunk, e.g. with an upload id from your server.
+  - `onErrorChunk(item, chunk, response, status, headers)` - when a chunk fails; the item then fails as usual (`onErrorItem`, `onCompleteItem`) and no more chunks are sent.
+  - `resumeItem(item)` - uploads a failed or cancelled item again, starting from the chunk that did not complete. Use it to retry, e.g. from `onErrorItem` (with `removeAfterUpload` off, so failed items stay in the queue):
+
+  ```typescript
+  const retries = new Map<FileItem, number>();
+  uploader.onErrorItem = (item, response, status) => {
+    const count = retries.get(item) ?? 0;
+    if (status >= 500 && count < 3) {
+      retries.set(item, count + 1);
+      setTimeout(() => uploader.resumeItem(item), 1000);
+    }
+  };
+  ```
+
+  Each attempt reports the item again (`onErrorItem`, `onCompleteItem`, and `onCompleteAll` when the queue is empty) and restarts its progress.
+
+  - `getChunk(item)` - the chunk being sent, or the last one sent.
+
+  `onBuildItemForm` and the `response` emitter fire once per chunk; the other item callbacks once per file. `item.cancel()` stops the remaining chunks; it has no effect once the last chunk has completed, and an item whose chunk failed stays failed. If a chunk callback throws, the item fails with status 0 and the error is rethrown; resuming then resends that chunk.
+
 # Troubleshooting
 
 Please follow these guidelines when reporting bugs and feature requests:

apps/demo/src/doc.md

@@ -45,3 +45,40 @@ import { FileSelectDirective, FileDropDirective, FileUploader } from 'ng2-file-u
   See using in [ts demo](https://github.com/valor-software/ng2-file-upload/blob/master/demo/components/file-upload/simple-demo.ts) and
   [html demo](https://github.com/valor-software/ng2-file-upload/blob/master/demo/components/file-upload/simple-demo.html)
   - `onFileDrop` - it fires after a file has been dropped on a Drop Area; you can pass in `$event` to get the list of files that were dropped. i.e. `(onFileDrop)="dropped($event)"`
+
+## Chunked uploads
+
+  Use `ChunkedFileUploader` (a `FileUploader` subclass) instead of `FileUploader` to send large files in several requests. It works with the same directives, options and callbacks; without `chunkSize` it behaves exactly like `FileUploader`.
+
+  ```typescript
+  uploader = new ChunkedFileUploader({ url: URL, chunkSize: 2 * 1024 * 1024 });
+  ```
+
+  Additional options:
+
+  1. `chunkSize` - Bytes per request. Multipart requests send the chunk as the file field, preceded by `chunkIndex` and `totalChunks` fields; with `disableMultipart` the raw chunk is sent with a `Content-Range` header (`formatDataFunction` is not used).
+  2. `chunkIndexParam` / `totalChunksParam` - Names of those form fields. Default to `chunkIndex` and `totalChunks`.
+
+  Additional callbacks and methods:
+
+  - `onBeforeUploadChunk(item, chunk)` - before each chunk request.
+  - `onSuccessChunk(item, chunk, response, status, headers)` - after each successful chunk. Change `item.url`, `item.method` or `item.headers` here to target the next chunk, e.g. with an upload id from your server.
+  - `onErrorChunk(item, chunk, response, status, headers)` - when a chunk fails; the item then fails as usual (`onErrorItem`, `onCompleteItem`) and no more chunks are sent.
+  - `resumeItem(item)` - uploads a failed or cancelled item again, starting from the chunk that did not complete. Use it to retry, e.g. from `onErrorItem` (with `removeAfterUpload` off, so failed items stay in the queue):
+
+  ```typescript
+  const retries = new Map<FileItem, number>();
+  uploader.onErrorItem = (item, response, status) => {
+    const count = retries.get(item) ?? 0;
+    if (status >= 500 && count < 3) {
+      retries.set(item, count + 1);
+      setTimeout(() => uploader.resumeItem(item), 1000);
+    }
+  };
+  ```
+
+  Each attempt reports the item again (`onErrorItem`, `onCompleteItem`, and `onCompleteAll` when the queue is empty) and restarts its progress.
+
+  - `getChunk(item)` - the chunk being sent, or the last one sent.
+
+  `onBuildItemForm` and the `response` emitter fire once per chunk; the other item callbacks once per file. `item.cancel()` stops the remaining chunks; it has no effect once the last chunk has completed, and an item whose chunk failed stays failed. If a chunk callback throws, the item fails with status 0 and the error is rethrown; resuming then resends that chunk.

libs/ng2-file-upload/file-upload/chunked-file-uploader.class.ts

@@ -0,0 +1,190 @@
+import { FileItem } from './file-item.class';
+import { FileUploader, FileUploaderOptions, ParsedResponseHeaders } from './file-uploader.class';
+
+export interface ChunkedFileUploaderOptions extends FileUploaderOptions {
+  chunkSize?: number;
+  chunkIndexParam?: string;
+  totalChunksParam?: string;
+}
+
+export interface FileChunk {
+  index: number;
+  total: number;
+  start: number;
+  end: number;
+  blob: Blob;
+}
+
+interface ChunkState {
+  chunk?: FileChunk;
+  resume?: FileChunk;
+  cancelled?: boolean;
+}
+
+export class ChunkedFileUploader extends FileUploader {
+  declare options: ChunkedFileUploaderOptions;
+
+  protected _chunks = new WeakMap<FileItem, ChunkState>();
+
+  constructor(options: ChunkedFileUploaderOptions) {
+    super(options);
+  }
+
+  setOptions(options: ChunkedFileUploaderOptions): void {
+    super.setOptions(options);
+  }
+
+  getChunk(item: FileItem): FileChunk | undefined {
+    return this._chunks.get(item)?.chunk;
+  }
+
+  resumeItem(item: FileItem): void {
+    const state = this._chunks.get(item);
+    if (item.isUploading || this.getIndexOfItem(item) < 0) {
+      return;
+    }
+    if (state?.chunk && !item.isSuccess) {
+      state.resume = state.chunk;
+    }
+    item.upload();
+  }
+
+  cancelItem(value: FileItem): void {
+    const item = this.queue[ this.getIndexOfItem(value) ];
+    const state = item && this._chunks.get(item);
+    if (state && item.isUploading) {
+      state.cancelled = true;
+      item._xhr?.abort();
+
+      return;
+    }
+    super.cancelItem(value);
+  }
+
+  onBeforeUploadChunk(item: FileItem, chunk: FileChunk): any {
+    return { item, chunk };
+  }
+
+  onSuccessChunk(item: FileItem, chunk: FileChunk, response: string, status: number, headers: ParsedResponseHeaders): any {
+    return { item, chunk, response, status, headers };
+  }
+
+  onErrorChunk(item: FileItem, chunk: FileChunk, response: string, status: number, headers: ParsedResponseHeaders): any {
+    return { item, chunk, response, status, headers };
+  }
+
+  protected _xhrTransport(item: FileItem): any {
+    const chunkSize = Math.floor(this.options.chunkSize || 0);
+    if (chunkSize <= 0) {
+      return super._xhrTransport(item);
+    }
+    const resume = this._chunks.get(item)?.resume;
+    this._chunks.set(item, {});
+    this._onBeforeUploadItem(item);
+
+    if (typeof item._file.size !== 'number') {
+      throw new TypeError('The file specified is no longer valid');
+    }
+    const total = Math.max(1, Math.ceil(item._file.size / chunkSize));
+    this._sendChunk(item, resume || this._sliceChunk(item, 0, 0, chunkSize, total));
+  }
+
+  protected _sliceChunk(item: FileItem, index: number, start: number, length: number, total: number): FileChunk {
+    const end = Math.min(start + length, item._file.size);
+
+    return { index, total, start, end, blob: item._file.slice(start, end, item._file.type) };
+  }
+
+  protected _sendChunk(item: FileItem, chunk: FileChunk): void {
+    const state = this._chunks.get(item) as ChunkState;
+    const xhr = item._xhr = new XMLHttpRequest();
+    state.chunk = chunk;
+    this.onBeforeUploadChunk(item, chunk);
+    const sendable = this.options.disableMultipart ? chunk.blob : this._buildFormData(item, chunk.blob, {
+      [ this.options.chunkIndexParam || 'chunkIndex' ]: chunk.index.toString(),
+      [ this.options.totalChunksParam || 'totalChunks' ]: chunk.total.toString()
+    });
+    if (state.cancelled) {
+      this._finishItem(item, '_onCancelItem', '', 0, {});
+
+      return;
+    }
+
+    xhr.upload.onprogress = (event: any) => {
+      const sent = event.lengthComputable ? event.loaded / event.total * (chunk.end - chunk.start) : 0;
+      this._onProgressItem(item, item._file.size ? Math.round((chunk.start + sent) * 100 / item._file.size) : 0);
+    };
+    xhr.onload = () => this._onChunkDone(item, chunk, xhr, this._isSuccessCode(xhr.status));
+    xhr.onerror = () => this._onChunkDone(item, chunk, xhr, false);
+    xhr.onabort = () => {
+      state.cancelled = true;
+      this._onChunkDone(item, chunk, xhr, false);
+    };
+    this._openRequest(xhr, item);
+    const hasContentRange = [ ...(this.options.headers || []), ...item.headers ].some(header => header.name.toLowerCase() === 'content-range');
+    if (this.options.disableMultipart && !hasContentRange) {
+      const range = chunk.end > chunk.start ? `${ chunk.start }-${ chunk.end - 1 }` : '*';
+      xhr.setRequestHeader('Content-Range', `bytes ${ range }/${ item._file.size }`);
+    }
+    xhr.send(sendable);
+    this._render();
+  }
+
+  protected _onChunkDone(item: FileItem, chunk: FileChunk, xhr: XMLHttpRequest, isSuccess: boolean): void {
+    const state = this._chunks.get(item) as ChunkState;
+    const headers = this._parseHeaders(xhr.getAllResponseHeaders());
+    const response = this._transformResponse(xhr.response);
+    if (state.cancelled) {
+      this._finishItem(item, '_onCancelItem', response, xhr.status, headers);
+
+      return;
+    }
+    if (!isSuccess) {
+      if (this._runChunkHook(item, () => this.onErrorChunk(item, chunk, response, xhr.status, headers))) {
+        this._finishItem(item, '_onErrorItem', response, xhr.status, headers);
+      }
+
+      return;
+    }
+    if (!this._runChunkHook(item, () => this.onSuccessChunk(item, chunk, response, xhr.status, headers))) {
+      return;
+    }
+    if (chunk.index + 1 >= chunk.total) {
+      this._finishItem(item, '_onSuccessItem', response, xhr.status, headers);
+
+      return;
+    }
+    const next = this._sliceChunk(item, chunk.index + 1, chunk.end, chunk.end - chunk.start, chunk.total);
+    if (state.cancelled) {
+      state.chunk = next;
+      this._finishItem(item, '_onCancelItem', response, xhr.status, headers);
+
+      return;
+    }
+    this._runChunkHook(item, () => this._sendChunk(item, next));
+  }
+
+  // hooks after the first chunk run in XHR callbacks: fail the item instead of stalling the queue, then rethrow
+  protected _runChunkHook(item: FileItem, hook: () => void): boolean {
+    try {
+      hook();
+
+      return true;
+    } catch (e) {
+      if (item.isUploading) {
+        this._finishItem(item, '_onErrorItem', '', 0, {});
+      }
+      setTimeout(() => {
+        throw e;
+      });
+
+      return false;
+    }
+  }
+
+  protected _finishItem(item: FileItem, method: '_onSuccessItem' | '_onErrorItem' | '_onCancelItem', response: string,
+                        status: number, headers: ParsedResponseHeaders): void {
+    this[ method ](item, response, status, headers);
+    this._onCompleteItem(item, response, status, headers);
+  }
+}

libs/ng2-file-upload/file-upload/file-uploader.class.ts

@@ -299,9 +299,6 @@ export class FileUploader {
   }
 
   protected _xhrTransport(item: FileItem): any {
-    // tslint:disable-next-line:no-this-assignment
-    // eslint-disable-next-line @typescript-eslint/no-this-alias
-    const that = this;
     const xhr = item._xhr = new XMLHttpRequest();
     let sendable: any;
     this._onBeforeUploadItem(item);
@@ -310,28 +307,7 @@ export class FileUploader {
       throw new TypeError('The file specified is no longer valid');
     }
     if (!this.options.disableMultipart) {
-      sendable = new FormData();
-      this._onBuildItemForm(item, sendable);
-      const appendFile = () => sendable.append(item.alias, item._file, item.file.name);
-      if (!this.options.parametersBeforeFiles) {
-        appendFile();
-      }
-
-      // For AWS, Additional Parameters must come BEFORE Files
-      if (this.options.additionalParameter !== undefined) {
-        Object.keys(this.options.additionalParameter).forEach((key: string) => {
-          let paramVal = this.options.additionalParameter?.[ key ];
-          // Allow an additional parameter to include the filename
-          if (typeof paramVal === 'string' && paramVal.indexOf('{{file_name}}') >= 0 && item.file?.name) {
-            paramVal = paramVal.replace('{{file_name}}', item.file.name);
-          }
-          sendable.append(key, paramVal);
-        });
-      }
-
-      if (appendFile && this.options.parametersBeforeFiles) {
-        appendFile();
-      }
+      sendable = this._buildFormData(item, item._file);
     } else {
       if (this.options.formatDataFunction) {
         sendable = this.options.formatDataFunction(item);
@@ -362,6 +338,49 @@ export class FileUploader {
       this._onCancelItem(item, response, xhr.status, headers);
       this._onCompleteItem(item, response, xhr.status, headers);
     };
+    this._openRequest(xhr, item);
+    if (this.options.formatDataFunctionIsAsync) {
+      sendable.then(
+        (result: any) => xhr.send(JSON.stringify(result))
+      );
+    } else {
+      xhr.send(sendable);
+    }
+    this._render();
+  }
+
+  protected _buildFormData(item: FileItem, file: Blob, extraParameters: { [ key: string ]: string } = {}): FormData {
+    const sendable = new FormData();
+    this._onBuildItemForm(item, sendable);
+    Object.keys(extraParameters).forEach((key: string) => sendable.append(key, extraParameters[ key ]));
+    const appendFile = () => sendable.append(item.alias as string, file, item.file.name);
+    if (!this.options.parametersBeforeFiles) {
+      appendFile();
+    }
+
+    // For AWS, Additional Parameters must come BEFORE Files
+    if (this.options.additionalParameter !== undefined) {
+      Object.keys(this.options.additionalParameter).forEach((key: string) => {
+        let paramVal = this.options.additionalParameter?.[ key ];
+        // Allow an additional parameter to include the filename
+        if (typeof paramVal === 'string' && paramVal.indexOf('{{file_name}}') >= 0 && item.file?.name) {
+          paramVal = paramVal.replace('{{file_name}}', item.file.name);
+        }
+        sendable.append(key, paramVal);
+      });
+    }
+
+    if (appendFile && this.options.parametersBeforeFiles) {
+      appendFile();
+    }
+
+    return sendable;
+  }
+
+  protected _openRequest(xhr: XMLHttpRequest, item: FileItem): void {
+    // tslint:disable-next-line:no-this-assignment
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    const that = this;
     if (item.method && item.url) {
       xhr.open(item.method, item.url, true);
     }
@@ -384,14 +403,6 @@ export class FileUploader {
         that.response.emit(xhr.responseText);
       }
     };
-    if (this.options.formatDataFunctionIsAsync) {
-      sendable.then(
-        (result: any) => xhr.send(JSON.stringify(result))
-      );
-    } else {
-      xhr.send(sendable);
-    }
-    this._render();
   }
 
   protected _getTotalProgress(value = 0): number {

libs/ng2-file-upload/index.ts

@@ -1,4 +1,5 @@
 export * from './file-upload/file-uploader.class';
+export * from './file-upload/chunked-file-uploader.class';
 export * from './file-upload/file-item.class';
 export * from './file-upload/file-like-object.class';
 export * from './file-upload/file-like-object.class';