diff --git a/apps/backend/__tests__/integration/s3-sdk/index.spec.ts b/apps/backend/__tests__/integration/s3-sdk/index.spec.ts index dacd8d800..d0b0b72ef 100644 --- a/apps/backend/__tests__/integration/s3-sdk/index.spec.ts +++ b/apps/backend/__tests__/integration/s3-sdk/index.spec.ts @@ -415,4 +415,100 @@ describe('AWS S3 - SDK', () => { expect(result.Metadata?.cid).toBeDefined() }) }) + + // Raw HTTP requests that mimic the AWS CLI / botocore, which (unlike the JS + // SDK used above) does NOT send the `x-id` query param for GetObject/ + // PutObject and sends object bodies with no Content-Type header. These guard + // two regressions: + // 1. getS3Method must fall back to the HTTP method (GET->GetObject, + // PUT->PutObject) when `x-id` is absent — otherwise dispatch returns + // "Method not found". + // 2. The request body must be read as raw bytes regardless of Content-Type; + // a missing Content-Type previously left req.body as {} and broke + // uploads deep in the IPLD chunker. + describe('Raw HTTP requests (AWS CLI style: no x-id, no Content-Type)', () => { + const S3_BASE = `${BASE_PATH}/s3` + // handleS3Auth only needs an Authorization header containing + // `Credential=/`; AuthManager is mocked to return `user`. + const AUTH = + 'AWS4-HMAC-SHA256 Credential=clitestkey/20200101/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=deadbeef' + + // Passing a Uint8Array/Buffer body to fetch leaves Content-Type unset, + // reproducing the AWS CLI's behaviour. No `x-id` query param is added. + const rawS3 = (method: string, path: string, body?: Uint8Array) => + fetch(`${S3_BASE}${path}`, { + method, + headers: { Authorization: AUTH }, + // Cast: TS 5.7 types Buffer/Uint8Array as Uint8Array, + // which doesn't structurally match the DOM BodyInit union. A binary + // body still sends with no Content-Type, which is the point here. + body: body as unknown as BodyInit | undefined, + }) + + const CliBody = Buffer.from('hello from the aws cli') + + it('PutObject without x-id/Content-Type stores the object', async () => { + const res = await rawS3('PUT', '/cli-test/hello.txt', CliBody) + expect(res.status).toBe(200) + expect(res.headers.get('etag')).toMatch(MD5_ETAG_RE) + }, 15_000) + + it('GetObject without x-id returns the exact bytes', async () => { + const res = await rawS3('GET', '/cli-test/hello.txt') + expect(res.status).toBe(200) + const got = Buffer.from(await res.arrayBuffer()) + expect(got).toEqual(CliBody) + }, 15_000) + + it('multipart upload via raw requests round-trips (the original 500)', async () => { + const key = '/cli-test/mpu.bin' + const part1 = Buffer.from('AAAAAAAAAAAAAAAA') + const part2 = Buffer.from('BBBBBBBBBBBBBBBB') + + const create = await rawS3('POST', `${key}?uploads`) + expect(create.status).toBe(200) + const uploadId = (await create.text()).match( + /([^<]+)<\/UploadId>/, + )?.[1] + expect(uploadId).toBeDefined() + + // Parts must be uploaded sequentially (the chunker enforces ordering). + const up1 = await rawS3( + 'PUT', + `${key}?partNumber=1&uploadId=${uploadId}`, + part1, + ) + expect(up1.status).toBe(200) + const etag1 = up1.headers.get('etag')! + expect(etag1).toMatch(MD5_ETAG_RE) + + const up2 = await rawS3( + 'PUT', + `${key}?partNumber=2&uploadId=${uploadId}`, + part2, + ) + expect(up2.status).toBe(200) + const etag2 = up2.headers.get('etag')! + + // The part list in the body is used only to compute the composite ETag. + const completeBody = Buffer.from( + '' + + `${etag1}1` + + `${etag2}2` + + '', + ) + const complete = await rawS3( + 'POST', + `${key}?uploadId=${uploadId}`, + completeBody, + ) + expect(complete.status).toBe(200) + expect(complete.headers.get('etag')).toMatch(MULTIPART_ETAG_RE) + + const get = await rawS3('GET', key) + expect(get.status).toBe(200) + const got = Buffer.from(await get.arrayBuffer()) + expect(got).toEqual(Buffer.concat([part1, part2])) + }, 30_000) + }) }) diff --git a/apps/backend/__tests__/unit/core/s3.spec.ts b/apps/backend/__tests__/unit/core/s3.spec.ts index 70db1b4fa..e947a3e2e 100644 --- a/apps/backend/__tests__/unit/core/s3.spec.ts +++ b/apps/backend/__tests__/unit/core/s3.spec.ts @@ -369,4 +369,127 @@ describe('S3UseCases', () => { ) }) }) + + describe('listObjects', () => { + // dbLimit for delimiter listings is min(maxKeys * 10 + 100, 10_000), so + // maxKeys=2 yields dbLimit=120 — small enough to construct test data for. + const DELIMITER_DB_LIMIT = (maxKeys: number) => + Math.min(maxKeys * 10 + 100, 10_000) + + const makeListing = (key: string) => ({ + key, + cid: 'cid', + size: 0n, + lastModified: new Date(0), + }) + + it('advances continuation token past a folded CommonPrefix when the DB batch is exhausted inside one prefix group', async () => { + // Regression test for Cursor Bugbot finding on PR #696 / #709. + // + // Scenario: maxKeys=2, delimiter='/', and a single virtual directory + // ('big/') contains more keys than fit in one DB batch. Every fetched + // row folds into the same CommonPrefix, so the in-loop maxKeys cap is + // never hit and the loop exhausts the batch with isTruncated=false. + // The fallback branch must then set the continuation token to a value + // that sorts *after* every key in 'big/' — otherwise the next page + // re-scans the rest of that directory and emits 'big/' again. + const maxKeys = 2 + const dbLimit = DELIMITER_DB_LIMIT(maxKeys) + + // Fill the entire DB batch with keys that all fold into 'big/'. + const fullBatch = Array.from({ length: dbLimit }, (_, i) => + makeListing(`big/${String(i).padStart(6, '0')}`), + ) + + jest + .spyOn(s3ObjectMappingsRepository, 'listObjects') + .mockResolvedValue(fullBatch as any) + + const result = await S3UseCases.listObjects({ + bucket: 'my-bucket', + prefix: '', + delimiter: '/', + maxKeys, + continuationToken: null, + }) + + expect(result.commonPrefixes).toEqual(['big/']) + expect(result.objects).toEqual([]) + expect(result.isTruncated).toBe(true) + // Token must start with the folded prefix and sort strictly after every + // key inside it. `￿` (U+FFFF) is the sentinel chosen for this purpose. + expect(result.nextContinuationToken).toBe('big/￿') + // Sanity: the token sorts after the last key we returned in the batch. + expect( + result.nextContinuationToken! > fullBatch[fullBatch.length - 1].key, + ).toBe(true) + }) + + it('uses the raw last key as the token when the last scanned key did not fold into a prefix', async () => { + // If the DB batch is full but the last key has no delimiter occurrence + // after the prefix, there's no CommonPrefix to skip past — fall back to + // the raw last key, which is the safe pre-fix behaviour. + const maxKeys = 2 + const dbLimit = DELIMITER_DB_LIMIT(maxKeys) + + // Pad the batch with folded entries, but make the LAST one a top-level + // key with no delimiter after the prefix. + const batch = [ + ...Array.from({ length: dbLimit - 1 }, (_, i) => + makeListing(`folder/${String(i).padStart(6, '0')}`), + ), + makeListing('zzz-top-level'), + ] + + jest + .spyOn(s3ObjectMappingsRepository, 'listObjects') + .mockResolvedValue(batch as any) + + const result = await S3UseCases.listObjects({ + bucket: 'my-bucket', + prefix: '', + delimiter: '/', + maxKeys, + continuationToken: null, + }) + + expect(result.isTruncated).toBe(true) + // The last key doesn't fold into a CommonPrefix, so the token stays as + // the raw key — no sentinel needed. + expect(result.nextContinuationToken).toBe('zzz-top-level') + }) + + it('uses the raw last key as the token when no delimiter is set', async () => { + // Without a delimiter, the dbLimit is maxKeys + 1, and there are no + // CommonPrefixes to repeat — the safe fallback is just the last key. + const maxKeys = 2 + const dbLimit = maxKeys + 1 // = 3 + + const batch = [ + makeListing('a.txt'), + makeListing('b.txt'), + makeListing('c.txt'), + ] + + jest + .spyOn(s3ObjectMappingsRepository, 'listObjects') + .mockResolvedValue(batch as any) + + const result = await S3UseCases.listObjects({ + bucket: 'my-bucket', + prefix: '', + delimiter: null, + maxKeys, + continuationToken: null, + }) + + // maxKeys=2 ⇒ first two keys returned, third triggers truncation in + // buildListResult (not the fallback), token = key just returned. + expect(result.objects.map((o) => o.key)).toEqual(['a.txt', 'b.txt']) + expect(result.isTruncated).toBe(true) + expect(result.nextContinuationToken).toBe('b.txt') + // dbLimit branch shouldn't have triggered, so no sentinel appended. + expect(batch.length).toBe(dbLimit) + }) + }) }) diff --git a/apps/backend/__tests__/unit/repositories/nodes.spec.ts b/apps/backend/__tests__/unit/repositories/nodes.spec.ts index bb5329242..3b1a200fd 100644 --- a/apps/backend/__tests__/unit/repositories/nodes.spec.ts +++ b/apps/backend/__tests__/unit/repositories/nodes.spec.ts @@ -10,8 +10,9 @@ import { nodesRepository, Node, } from '../../../src/infrastructure/repositories/objects/nodes.js' +import { metadataRepository } from '../../../src/infrastructure/repositories/objects/metadata.js' import { dbMigration } from '../../utils/dbMigrate.js' -import { MetadataType } from '@autonomys/auto-dag-data' +import { MetadataType, OffchainMetadata } from '@autonomys/auto-dag-data' describe('Nodes Repository', () => { beforeAll(async () => { @@ -370,22 +371,22 @@ describe('Nodes Repository', () => { expect(result?.piece_offset).toBe(100) }) - it('should remove nodes by root CID', async () => { + it('should remove encoded_node only for published nodes by root CID', async () => { const rootCid = 'test-root-cid-remove' const nodes: Node[] = [ { - cid: 'test-cid-remove-1', + cid: 'test-cid-remove-published', root_cid: rootCid, head_cid: 'test-head-cid-remove', type: 'file', encoded_node: 'test-encoded-node-remove-1', piece_index: null, piece_offset: null, - block_published_on: null, + block_published_on: 100, tx_published_on: null, }, { - cid: 'test-cid-remove-2', + cid: 'test-cid-remove-unpublished', root_cid: rootCid, head_cid: 'test-head-cid-remove', type: 'file', @@ -399,13 +400,16 @@ describe('Nodes Repository', () => { await nodesRepository.saveNodes(nodes) await nodesRepository.removeNodeDataByRootCid(rootCid) - const results = await nodesRepository.getNodesByRootCid(rootCid) - const fullNodes = await Promise.all( - results.map((r) => nodesRepository.getNode(r.cid)), + + const publishedNode = await nodesRepository.getNode( + 'test-cid-remove-published', ) - fullNodes.forEach((n) => { - expect(n?.encoded_node).toBeNull() - }) + expect(publishedNode?.encoded_node).toBeNull() + + const unpublishedNode = await nodesRepository.getNode( + 'test-cid-remove-unpublished', + ) + expect(unpublishedNode?.encoded_node).toBe('test-encoded-node-remove-2') }) it('should get nodes by CIDs', async () => { @@ -466,6 +470,44 @@ describe('Nodes Repository', () => { expect(result?.block_published_on).toBe(12345) expect(result?.tx_published_on).toBe('tx-hash') + expect(result?.encoded_node).toBe('test-encoded-node-published') + }) + + it('should clear encoded_node on publish when metadata is already archived', async () => { + const rootCid = 'test-root-cid-publish-archived' + const headCid = 'test-head-cid-publish-archived' + const metadata: OffchainMetadata = { + totalSize: 100n, + type: 'file', + dataCid: 'test-data-cid-publish-archived', + totalChunks: 1, + chunks: [], + name: 'test-file-publish-archived', + } + + await metadataRepository.setMetadata(rootCid, headCid, metadata) + await metadataRepository.markAsArchived(headCid) + + const node: Node = { + cid: 'test-cid-publish-after-archive', + root_cid: rootCid, + head_cid: headCid, + type: 'file', + encoded_node: 'data-that-should-be-cleared', + piece_index: null, + piece_offset: null, + block_published_on: null, + tx_published_on: null, + } + + await nodesRepository.saveNode(node) + await nodesRepository.updateNodePublishedOn(node.cid, 99999, 'tx-recovery') + + const result = await nodesRepository.getNode(node.cid) + + expect(result?.block_published_on).toBe(99999) + expect(result?.tx_published_on).toBe('tx-recovery') + expect(result?.encoded_node).toBeNull() }) it('should get uploaded nodes by root CID', async () => { diff --git a/apps/backend/src/app/apis/download.ts b/apps/backend/src/app/apis/download.ts index 34e862e24..5911ed5ec 100644 --- a/apps/backend/src/app/apis/download.ts +++ b/apps/backend/src/app/apis/download.ts @@ -15,6 +15,26 @@ const createServer = async () => { logger.debug('Initializing download API server') const app = express() + if (config.express.corsAllowedOrigins) { + logger.debug( + 'Configuring CORS with allowed origins: %j', + config.express.corsAllowedOrigins, + ) + app.use( + cors({ + origin: config.express.corsAllowedOrigins, + }), + ) + } else { + logger.warn('CORS is not configured - no allowed origins specified, blocking cross-origin requests') + } + + // The S3 controller handles its own raw body parsing (binary object + // payloads). It is mounted before the JSON/urlencoded parsers below so those + // never run for /s3 — otherwise body-parser would set req.body to {} and the + // raw object bytes would be lost. + app.use('/s3', s3Controller) + app.use( express.json({ limit: config.express.requestSizeLimit, @@ -36,22 +56,7 @@ const createServer = async () => { config.express.requestSizeLimit, ) - if (config.express.corsAllowedOrigins) { - logger.debug( - 'Configuring CORS with allowed origins: %j', - config.express.corsAllowedOrigins, - ) - app.use( - cors({ - origin: config.express.corsAllowedOrigins, - }), - ) - } else { - logger.warn('CORS is not configured - no allowed origins specified, blocking cross-origin requests') - } - app.use('/downloads', downloadController) - app.use('/s3', s3Controller) app.use('/features', featuresController) logger.debug('Download controller mounted at /downloads') diff --git a/apps/backend/src/app/controllers/s3/http.ts b/apps/backend/src/app/controllers/s3/http.ts index fddc54eac..7b9c37223 100644 --- a/apps/backend/src/app/controllers/s3/http.ts +++ b/apps/backend/src/app/controllers/s3/http.ts @@ -55,6 +55,16 @@ const getS3Method = (req: Request) => { if (req.method === 'DELETE') { return 'DeleteObject' } + // Method-based fallbacks for clients that do not send the `x-id` query + // param (e.g. the AWS CLI / botocore). These must come after the more + // specific query-param checks above so multipart PUTs (uploadId + + // partNumber) and ListObjectsV2 GETs (list-type=2) are not misrouted. + if (req.method === 'GET') { + return 'GetObject' + } + if (req.method === 'PUT') { + return 'PutObject' + } } return s3Method } @@ -69,8 +79,13 @@ s3Controller.get( s3Controller.use( '/:key(*)', + // S3 object bodies are opaque binary payloads. Always read the request body + // as raw bytes regardless of (or the absence of) a Content-Type header. + // `type: '*/*'` is insufficient: type-is returns false when no Content-Type + // header is present, which the AWS CLI omits on PutObject/UploadPart — that + // left req.body as `{}` and broke uploads. raw({ - type: '*/*', + type: () => true, limit: '100mb', }), asyncSafeHandler(async (req: Request, res: Response) => { diff --git a/apps/backend/src/core/objects/publishingRecovery.ts b/apps/backend/src/core/objects/publishingRecovery.ts index 8a868f475..3ab0154ab 100644 --- a/apps/backend/src/core/objects/publishingRecovery.ts +++ b/apps/backend/src/core/objects/publishingRecovery.ts @@ -46,6 +46,22 @@ const processPublishingRecovery = async (): Promise => { const runRecoveryBatch = async (): Promise => { const maxPerCycle = config.publishingRecovery.maxObjectsPerCycle + // Surface objects that are stuck in an unrecoverable state — they have + // unpublished nodes whose encoded_node was already stripped (e.g. by + // archival before publishing completed). These cannot be recovered + // automatically and need operational attention. + const unrecoverable = + await nodesRepository.getUnrecoverablePublishingRootCids(maxPerCycle) + if (unrecoverable.length > 0) { + logger.warn( + 'Found %d objects with unrecoverable unpublished nodes (data stripped before publishing): %s', + unrecoverable.length, + unrecoverable + .map((r) => `${r.root_cid} (${r.unrecoverable_count} nodes)`) + .join(', '), + ) + } + const stuckRootCids = await nodesRepository.getStuckPublishingRootCids( maxPerCycle, config.publishingRecovery.stalenessThresholdBlocks, diff --git a/apps/backend/src/core/s3/index.ts b/apps/backend/src/core/s3/index.ts index be795565d..433bbf489 100644 --- a/apps/backend/src/core/s3/index.ts +++ b/apps/backend/src/core/s3/index.ts @@ -197,7 +197,31 @@ const listObjects = async ( // harmless, but silently dropping data is not. if (!isTruncated && allMatching.length === dbLimit) { isTruncated = true - nextContinuationToken = allMatching[allMatching.length - 1].key + const lastKey = allMatching[allMatching.length - 1].key + + // If the last scanned key folded into a CommonPrefix, the naive choice + // `lastKey` would land *inside* a virtual directory we've already + // represented in commonPrefixes. The next page's `key > token` query + // would then return the remaining keys in that directory, which would + // re-fold into — and re-emit — the same CommonPrefix entry. + // + // Advance the token past every key that could possibly fold into that + // prefix by appending a high-sort sentinel. `￿` (encoded as + // 0xEF 0xBF 0xBF in UTF-8) sorts after every realistic S3 key character, + // so `key > token` skips the rest of the directory and resumes at the + // first key that falls outside it. + if (delimiter) { + const afterPrefix = lastKey.slice(prefix.length) + const delimIdx = afterPrefix.indexOf(delimiter) + if (delimIdx >= 0) { + const lastFoldedPrefix = prefix + afterPrefix.slice(0, delimIdx + 1) + nextContinuationToken = lastFoldedPrefix + '￿' + } else { + nextContinuationToken = lastKey + } + } else { + nextContinuationToken = lastKey + } } return { diff --git a/apps/backend/src/infrastructure/repositories/objects/nodes.ts b/apps/backend/src/infrastructure/repositories/objects/nodes.ts index 3c61cd97c..9bc47a139 100644 --- a/apps/backend/src/infrastructure/repositories/objects/nodes.ts +++ b/apps/backend/src/infrastructure/repositories/objects/nodes.ts @@ -176,11 +176,22 @@ const setNodeArchivingData = async ({ }) } +/** + * Removes encoded_node data for all *published* nodes under a root_cid. + * + * Only nodes with `block_published_on IS NOT NULL` are stripped — their + * data is already immutably on-chain and the local copy is no longer + * needed. Unpublished nodes retain their encoded_node so the publishing + * recovery job can still re-enqueue them. + */ const removeNodeDataByRootCid = async (rootCid: string) => { const db = await getDatabase() return db.query({ - text: 'UPDATE nodes SET encoded_node = NULL WHERE root_cid = $1', + text: `UPDATE nodes + SET encoded_node = NULL + WHERE root_cid = $1 + AND block_published_on IS NOT NULL`, values: [rootCid], }) } @@ -212,7 +223,17 @@ const updateNodePublishedOn = async ( const db = await getDatabase() return db.query({ - text: 'UPDATE nodes SET block_published_on = $1, tx_published_on = $2 WHERE cid = $3', + text: `UPDATE nodes + SET block_published_on = $1, + tx_published_on = $2, + encoded_node = CASE + WHEN EXISTS ( + SELECT 1 FROM metadata + WHERE head_cid = nodes.head_cid AND is_archived = true + ) THEN NULL + ELSE encoded_node + END + WHERE cid = $3`, values: [blockPublishedOn, txPublishedOn, cid], }) } @@ -421,6 +442,10 @@ const getFullyArchivedHeadCids = async ( * * The staleness filter prevents false positives on objects that are * still being actively published in the normal pipeline. + * + * Objects where every unpublished node has `encoded_node IS NULL` + * (i.e. archived before publishing completed) are excluded — they + * are un-publishable and would only cause repeated failures. */ const getStuckPublishingRootCids = async ( limit: number, @@ -436,6 +461,9 @@ const getStuckPublishingRootCids = async ( GROUP BY root_cid HAVING COUNT(block_published_on) > 0 AND COUNT(block_published_on) < COUNT(*) + AND COUNT(*) FILTER ( + WHERE block_published_on IS NULL AND encoded_node IS NOT NULL + ) > 0 AND MAX(block_published_on) + $2 < ( SELECT MAX(block_published_on) FROM nodes ) @@ -448,6 +476,8 @@ const getStuckPublishingRootCids = async ( /** * Returns CIDs of unpublished nodes for a given root_cid. + * Only returns nodes that still have encoded_node data — nodes whose + * data was removed (e.g. by archival) are un-publishable and excluded. */ const getUnpublishedNodeCidsByRootCid = async ( rootCid: string, @@ -461,12 +491,45 @@ const getUnpublishedNodeCidsByRootCid = async ( FROM nodes WHERE root_cid = $1 AND block_published_on IS NULL + AND encoded_node IS NOT NULL `, values: [rootCid], }) .then((e) => e.rows.map((r) => r.cid)) } +/** + * Returns root_cids that have nodes stuck in an unrecoverable state: + * `block_published_on IS NULL` (never published) AND `encoded_node IS NULL` + * (data already stripped, e.g. by a prior archival bug). + * + * These objects can never complete publishing from local state alone. + * The caller should surface them for operational attention. + */ +const getUnrecoverablePublishingRootCids = async ( + limit: number, +): Promise<{ root_cid: string; unrecoverable_count: number }[]> => { + const db = await getDatabase() + + return db + .query<{ root_cid: string; unrecoverable_count: number }>({ + text: ` + SELECT root_cid, + COUNT(*) FILTER ( + WHERE block_published_on IS NULL AND encoded_node IS NULL + )::int AS unrecoverable_count + FROM nodes + GROUP BY root_cid + HAVING COUNT(*) FILTER ( + WHERE block_published_on IS NULL AND encoded_node IS NULL + ) > 0 + LIMIT $1 + `, + values: [limit], + }) + .then((e) => e.rows) +} + export const nodesRepository = { getNode, getNodeCount, @@ -493,4 +556,5 @@ export const nodesRepository = { getFullyArchivedHeadCids, getStuckPublishingRootCids, getUnpublishedNodeCidsByRootCid, + getUnrecoverablePublishingRootCids, } diff --git a/apps/backend/src/infrastructure/services/upload/onchainPublisher/index.ts b/apps/backend/src/infrastructure/services/upload/onchainPublisher/index.ts index 00b09395d..3738002be 100644 --- a/apps/backend/src/infrastructure/services/upload/onchainPublisher/index.ts +++ b/apps/backend/src/infrastructure/services/upload/onchainPublisher/index.ts @@ -23,11 +23,30 @@ const publishNodes = async (cids: string[]) => { const repeatedNodes = nodes.filter((node) => publishedCidSet.has(node.cid)) await NodesUseCases.handleRepeatedNodes(repeatedNodes) - // Nodes that are not known as published anywhere should be sent for publishing - const publishingNodes = nodes.filter((node) => !publishedCidSet.has(node.cid)) + // Nodes that are not known as published anywhere should be sent for publishing. + // Filter out nodes whose encoded_node has been removed (e.g. after archival) — + // these are un-publishable and must not reach the Buffer.from() call. + const publishingNodes = nodes.filter( + (node) => !publishedCidSet.has(node.cid) && node.encoded_node != null, + ) + + const skippedNullCount = nodes.filter( + (node) => !publishedCidSet.has(node.cid) && node.encoded_node == null, + ).length + if (skippedNullCount > 0) { + logger.warn( + 'Skipped %d nodes with NULL encoded_node (archived before publishing)', + skippedNullCount, + ) + } + + if (publishingNodes.length === 0) { + logger.info('No publishable nodes remaining after filtering') + return + } const transactions = publishingNodes.map((node) => { - const buffer = Buffer.from(node.encoded_node, 'base64') + const buffer = Buffer.from(node.encoded_node!, 'base64') return { module: 'system', diff --git a/apps/frontend/__tests__/unit/services/bulkObjectDownload.spec.ts b/apps/frontend/__tests__/unit/services/bulkObjectDownload.spec.ts new file mode 100644 index 000000000..c69385989 --- /dev/null +++ b/apps/frontend/__tests__/unit/services/bulkObjectDownload.spec.ts @@ -0,0 +1,228 @@ +jest.mock('@auto-drive/models', () => ({ + isInsecure: (tags: string[]) => tags.includes('insecure'), + isBanned: (tags: string[]) => tags.includes('banned'), +})); + +import { + BulkDownloadItem, + EncryptionContext, + hasEncryption, + itemIsRunnable, + resolveEncryptionOptions, + shouldSkipEncrypted, + shouldSkipInsecure, +} from '../../../src/services/bulkObjectDownload'; + +const baseInformation = { + tags: [] as string[], + metadata: { + dataCid: 'bafkr6itestcid', + name: 'test.txt', + type: 'file', + totalSize: 12, + uploadOptions: {}, + }, +} as unknown as BulkDownloadItem['information']; + +interface InformationOverrides { + tags?: string[]; + metadata?: Record; +} + +const makeItem = ( + overrides: Partial = {}, + informationOverrides: InformationOverrides = {}, +): BulkDownloadItem => ({ + cid: 'bafkr6itestcid', + status: 'pending', + ...overrides, + information: { + ...baseInformation, + ...informationOverrides, + metadata: { + ...baseInformation!.metadata, + ...(informationOverrides.metadata ?? {}), + }, + } as BulkDownloadItem['information'], +}); + +const encryptionMetadata = { + uploadOptions: { + encryption: { algorithm: 'aes-256-gcm' }, + }, +}; + +describe('hasEncryption', () => { + it('returns true when the upload options declare an encryption algorithm', () => { + expect(hasEncryption(makeItem({}, { metadata: encryptionMetadata }))).toBe( + true, + ); + }); + + it('returns false when no encryption algorithm is declared', () => { + expect(hasEncryption(makeItem())).toBe(false); + }); + + it('returns false for items without metadata', () => { + expect( + hasEncryption({ cid: 'x', status: 'failed' } as BulkDownloadItem), + ).toBe(false); + }); +}); + +describe('itemIsRunnable', () => { + it('is true for pending items with information', () => { + expect(itemIsRunnable(makeItem({ status: 'pending' }))).toBe(true); + }); + + it('is false for non-pending items', () => { + for (const status of [ + 'skipped', + 'checking', + 'preparing', + 'downloading', + 'completed', + 'failed', + ] as const) { + expect(itemIsRunnable(makeItem({ status }))).toBe(false); + } + }); + + it('is false when information is missing (metadata load failed)', () => { + expect( + itemIsRunnable({ + cid: 'x', + status: 'pending', + } as BulkDownloadItem), + ).toBe(false); + }); +}); + +describe('shouldSkipInsecure', () => { + it('skips insecure items when the user has not confirmed', () => { + const item = makeItem({}, { tags: ['insecure'] }); + expect(shouldSkipInsecure(item, false)).toBe(true); + }); + + it('does not skip insecure items once confirmed', () => { + const item = makeItem({}, { tags: ['insecure'] }); + expect(shouldSkipInsecure(item, true)).toBe(false); + }); + + it('does not skip non-insecure items', () => { + expect(shouldSkipInsecure(makeItem(), false)).toBe(false); + }); + + it('does not skip when information is missing', () => { + expect( + shouldSkipInsecure( + { cid: 'x', status: 'pending' } as BulkDownloadItem, + false, + ), + ).toBe(false); + }); +}); + +describe('shouldSkipEncrypted', () => { + const encryptedItem = makeItem({}, { metadata: encryptionMetadata }); + + it('skips encrypted items when the user chose skip and has no default password', () => { + const ctx: EncryptionContext = { + defaultPassword: undefined, + encryptionChoice: 'skip', + sharedPassword: '', + }; + expect(shouldSkipEncrypted(encryptedItem, ctx)).toBe(true); + }); + + it('does not skip encrypted items when a default password is set', () => { + const ctx: EncryptionContext = { + defaultPassword: 'pw', + encryptionChoice: 'skip', + sharedPassword: '', + }; + expect(shouldSkipEncrypted(encryptedItem, ctx)).toBe(false); + }); + + it('does not skip when user chose download-encrypted', () => { + const ctx: EncryptionContext = { + defaultPassword: undefined, + encryptionChoice: 'download-encrypted', + sharedPassword: '', + }; + expect(shouldSkipEncrypted(encryptedItem, ctx)).toBe(false); + }); + + it('does not skip non-encrypted items even with skip choice', () => { + const ctx: EncryptionContext = { + defaultPassword: undefined, + encryptionChoice: 'skip', + sharedPassword: '', + }; + expect(shouldSkipEncrypted(makeItem(), ctx)).toBe(false); + }); +}); + +describe('resolveEncryptionOptions', () => { + const encryptedItem = makeItem({}, { metadata: encryptionMetadata }); + + it('returns no password and no skip for non-encrypted items', () => { + expect( + resolveEncryptionOptions(makeItem(), { + defaultPassword: 'pw', + encryptionChoice: 'shared-password', + sharedPassword: 'shared', + }), + ).toEqual({ password: undefined, skipDecryption: false }); + }); + + it('uses the default password when one is set', () => { + expect( + resolveEncryptionOptions(encryptedItem, { + defaultPassword: 'defaultpw', + encryptionChoice: null, + sharedPassword: '', + }), + ).toEqual({ password: 'defaultpw', skipDecryption: false }); + }); + + it('returns skipDecryption=true when user chose download-encrypted', () => { + expect( + resolveEncryptionOptions(encryptedItem, { + defaultPassword: undefined, + encryptionChoice: 'download-encrypted', + sharedPassword: 'ignored', + }), + ).toEqual({ password: undefined, skipDecryption: true }); + }); + + it('uses the shared password when user chose shared-password', () => { + expect( + resolveEncryptionOptions(encryptedItem, { + defaultPassword: undefined, + encryptionChoice: 'shared-password', + sharedPassword: 'sharedpw', + }), + ).toEqual({ password: 'sharedpw', skipDecryption: false }); + }); + + it('falls back to no-password/no-skip when user chose skip', () => { + expect( + resolveEncryptionOptions(encryptedItem, { + defaultPassword: undefined, + encryptionChoice: 'skip', + sharedPassword: '', + }), + ).toEqual({ password: undefined, skipDecryption: false }); + }); + + it('prefers default password over an explicit shared-password choice', () => { + expect( + resolveEncryptionOptions(encryptedItem, { + defaultPassword: 'defaultpw', + encryptionChoice: 'shared-password', + sharedPassword: 'sharedpw', + }), + ).toEqual({ password: 'defaultpw', skipDecryption: false }); + }); +}); diff --git a/apps/frontend/__tests__/unit/services/objectDownloadFlow.spec.ts b/apps/frontend/__tests__/unit/services/objectDownloadFlow.spec.ts new file mode 100644 index 000000000..0325867e2 --- /dev/null +++ b/apps/frontend/__tests__/unit/services/objectDownloadFlow.spec.ts @@ -0,0 +1,273 @@ +jest.mock('utils/auth', () => ({ + getAuthSession: jest.fn(), +})); + +jest.mock('@auto-drive/models', () => ({ + AsyncDownloadStatus: { + Failed: 'failed', + Dismissed: 'dismissed', + }, + DownloadStatus: { + Cached: 'cached', + NotCached: 'not-cached', + }, +})); + +import { AsyncDownloadStatus, DownloadStatus } from '@auto-drive/models'; +import { getAuthSession } from 'utils/auth'; +import { runObjectDownloadFlow } from '../../../src/services/objectDownloadFlow'; + +const metadata = { + dataCid: 'bafkr6itestcid', + name: 'test.txt', + type: 'file', + totalSize: 12, + uploadOptions: {}, +}; + +const createDependencies = () => { + const api = { + checkDownloadStatus: jest.fn(), + createAsyncDownload: jest.fn(), + }; + const downloadService = { + fetchFile: jest.fn(), + }; + + return { api, downloadService }; +}; + +describe('runObjectDownloadFlow', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('downloads immediately when the object is already cached', async () => { + (getAuthSession as jest.Mock).mockResolvedValue({ + accessToken: 'token', + authProvider: 'google', + }); + const { api, downloadService } = createDependencies(); + api.checkDownloadStatus.mockResolvedValue(DownloadStatus.Cached); + + await runObjectDownloadFlow({ + api: api as never, + downloadService, + metadata: metadata as never, + password: 'secret', + onPhaseChange: jest.fn(), + }); + + expect(api.checkDownloadStatus).toHaveBeenCalledWith(metadata.dataCid); + expect(api.createAsyncDownload).not.toHaveBeenCalled(); + expect(downloadService.fetchFile).toHaveBeenCalledWith(metadata.dataCid, { + password: 'secret', + skipDecryption: false, + onProgress: undefined, + }); + }); + + it('creates an async download for uncached authenticated objects before downloading', async () => { + (getAuthSession as jest.Mock).mockResolvedValue({ + accessToken: 'token', + authProvider: 'google', + }); + const { api, downloadService } = createDependencies(); + api.checkDownloadStatus + .mockResolvedValueOnce(DownloadStatus.NotCached) + .mockResolvedValueOnce(DownloadStatus.Cached); + const onPhaseChange = jest.fn(); + const onAsyncDownloadsRefresh = jest.fn(); + + await runObjectDownloadFlow({ + api: api as never, + downloadService, + metadata: metadata as never, + asyncPollIntervalMs: 0, + onPhaseChange, + onAsyncDownloadsRefresh, + }); + + expect(api.createAsyncDownload).toHaveBeenCalledWith(metadata.dataCid); + expect(onAsyncDownloadsRefresh).toHaveBeenCalled(); + expect(onPhaseChange.mock.calls.map(([phase]) => phase)).toEqual([ + 'checking', + 'preparing', + 'downloading', + 'completed', + ]); + expect(downloadService.fetchFile).toHaveBeenCalledTimes(1); + }); + + it('falls back to direct download when async creation fails', async () => { + (getAuthSession as jest.Mock).mockResolvedValue({ + accessToken: 'token', + authProvider: 'google', + }); + const { api, downloadService } = createDependencies(); + api.checkDownloadStatus.mockResolvedValueOnce(DownloadStatus.NotCached); + api.createAsyncDownload.mockRejectedValue(new Error('queue unavailable')); + + await runObjectDownloadFlow({ + api: api as never, + downloadService, + metadata: metadata as never, + asyncPollIntervalMs: 0, + }); + + expect(downloadService.fetchFile).toHaveBeenCalledTimes(1); + }); + + it('surfaces async preparation failures from the async downloads store', async () => { + (getAuthSession as jest.Mock).mockResolvedValue({ + accessToken: 'token', + authProvider: 'google', + }); + const { api, downloadService } = createDependencies(); + api.checkDownloadStatus + .mockResolvedValueOnce(DownloadStatus.NotCached) + .mockResolvedValueOnce(DownloadStatus.NotCached); + + await expect( + runObjectDownloadFlow({ + api: api as never, + downloadService, + metadata: metadata as never, + asyncPollIntervalMs: 0, + getAsyncDownloads: () => [ + { + cid: metadata.dataCid, + status: AsyncDownloadStatus.Failed, + errorMessage: 'Gateway timed out', + }, + ], + }), + ).rejects.toThrow('Gateway timed out'); + + expect(downloadService.fetchFile).not.toHaveBeenCalled(); + }); + + it('downloads directly for anonymous users', async () => { + (getAuthSession as jest.Mock).mockResolvedValue(null); + const { api, downloadService } = createDependencies(); + + await runObjectDownloadFlow({ + api: api as never, + downloadService, + metadata: metadata as never, + }); + + expect(api.checkDownloadStatus).not.toHaveBeenCalled(); + expect(api.createAsyncDownload).not.toHaveBeenCalled(); + expect(downloadService.fetchFile).toHaveBeenCalledTimes(1); + }); + + it('throws ObjectDownloadPreparationError when async polling times out', async () => { + (getAuthSession as jest.Mock).mockResolvedValue({ + accessToken: 'token', + authProvider: 'google', + }); + const { api, downloadService } = createDependencies(); + // initial check → NotCached triggers async creation; every subsequent + // poll also returns NotCached, exhausting maxAsyncPollCount. + api.checkDownloadStatus.mockResolvedValue(DownloadStatus.NotCached); + + await expect( + runObjectDownloadFlow({ + api: api as never, + downloadService, + metadata: metadata as never, + asyncPollIntervalMs: 0, + maxAsyncPollCount: 3, + }), + ).rejects.toThrow('Download preparation timed out'); + + expect(api.createAsyncDownload).toHaveBeenCalledTimes(1); + expect(downloadService.fetchFile).not.toHaveBeenCalled(); + }); + + it('aborts mid-poll when the AbortSignal fires', async () => { + (getAuthSession as jest.Mock).mockResolvedValue({ + accessToken: 'token', + authProvider: 'google', + }); + const { api, downloadService } = createDependencies(); + api.checkDownloadStatus.mockResolvedValue(DownloadStatus.NotCached); + + const controller = new AbortController(); + const flow = runObjectDownloadFlow({ + api: api as never, + downloadService, + metadata: metadata as never, + asyncPollIntervalMs: 100, + maxAsyncPollCount: 10, + signal: controller.signal, + }); + + // give the flow a chance to enter its first delay() + await Promise.resolve(); + await Promise.resolve(); + controller.abort(); + + await expect(flow).rejects.toThrow('Download aborted'); + expect(downloadService.fetchFile).not.toHaveBeenCalled(); + }); + + it('aborts after fetchFile completes if signal fired mid-download', async () => { + (getAuthSession as jest.Mock).mockResolvedValue(null); + const { api, downloadService } = createDependencies(); + + const controller = new AbortController(); + downloadService.fetchFile.mockImplementation(() => { + controller.abort(); + return Promise.resolve(); + }); + + const onPhaseChange = jest.fn(); + + await expect( + runObjectDownloadFlow({ + api: api as never, + downloadService, + metadata: metadata as never, + signal: controller.signal, + onPhaseChange, + }), + ).rejects.toThrow('Download aborted'); + + expect(downloadService.fetchFile).toHaveBeenCalledTimes(1); + expect(onPhaseChange).toHaveBeenCalledWith('downloading'); + expect(onPhaseChange).not.toHaveBeenCalledWith('completed'); + }); + + it('tolerates a transient poll-cycle error and continues polling', async () => { + (getAuthSession as jest.Mock).mockResolvedValue({ + accessToken: 'token', + authProvider: 'google', + }); + const { api, downloadService } = createDependencies(); + // 1st call (pre-async): NotCached + // 2nd call (1st poll): rejects with transient error → swallowed + // 3rd call (2nd poll): Cached → flow proceeds to download + api.checkDownloadStatus + .mockResolvedValueOnce(DownloadStatus.NotCached) + .mockRejectedValueOnce(new Error('network blip')) + .mockResolvedValueOnce(DownloadStatus.Cached); + + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + + await runObjectDownloadFlow({ + api: api as never, + downloadService, + metadata: metadata as never, + asyncPollIntervalMs: 0, + maxAsyncPollCount: 5, + }); + + expect(downloadService.fetchFile).toHaveBeenCalledTimes(1); + expect(warnSpy).toHaveBeenCalled(); + expect(warnSpy.mock.calls[0][0]).toContain('poll cycle'); + + warnSpy.mockRestore(); + }); +}); diff --git a/apps/frontend/src/components/molecules/BulkObjectDownloadModal.tsx b/apps/frontend/src/components/molecules/BulkObjectDownloadModal.tsx new file mode 100644 index 000000000..9a3ff1d94 --- /dev/null +++ b/apps/frontend/src/components/molecules/BulkObjectDownloadModal.tsx @@ -0,0 +1,660 @@ +import { + Dialog, + DialogPanel, + DialogTitle, + Transition, + TransitionChild, +} from '@headlessui/react'; +import { + Fragment, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; +import { + isBanned, + isInsecure, + ObjectInformation, + ObjectStatus, +} from '@auto-drive/models'; +import { Button } from '@auto-drive/ui'; +import toast from 'react-hot-toast'; +import { useNetwork } from 'contexts/network'; +import { useEncryptionStore } from 'globalStates/encryption'; +import { InvalidDecryptKey } from 'utils/file'; +import { + ObjectDownloadAbortedError, + runObjectDownloadFlow, +} from 'services/objectDownloadFlow'; +import { + BulkDownloadItem, + BulkDownloadStatus, + EncryptionChoice, + hasEncryption, + itemIsRunnable, + resolveEncryptionOptions as resolveBulkEncryptionOptions, + shouldSkipEncrypted, + shouldSkipInsecure, +} from 'services/bulkObjectDownload'; +import { formatBytes } from 'utils/number'; +import { shortenString } from 'utils/misc'; +import { useUserAsyncDownloadsStore } from '../organisms/UserAsyncDownloads/state'; + +const toastId = 'bulk-object-download-modal'; + +const statusLabel: Record = { + pending: 'Pending', + skipped: 'Skipped', + checking: 'Checking', + preparing: 'Preparing', + downloading: 'Downloading', + completed: 'Completed', + failed: 'Failed', +}; + +const itemName = (item: BulkDownloadItem) => + item.information?.metadata.name ?? item.cid; + +export const BulkObjectDownloadModal = ({ + cids, + isOpen, + onClose, + onComplete, +}: { + cids: string[]; + isOpen: boolean; + onClose: () => void; + onComplete: () => void; +}) => { + const network = useNetwork(); + const defaultPassword = useEncryptionStore((store) => store.password); + const updateAsyncDownloads = useUserAsyncDownloadsStore((e) => e.update); + const addPendingAutoDownload = useUserAsyncDownloadsStore( + (e) => e.addPendingAutoDownload, + ); + const [items, setItems] = useState([]); + const [isLoadingMetadata, setIsLoadingMetadata] = useState(false); + const [hasConfirmedInsecure, setHasConfirmedInsecure] = useState(false); + const [encryptionChoice, setEncryptionChoice] = + useState(null); + const [sharedPassword, setSharedPassword] = useState(''); + const [isRunning, setIsRunning] = useState(false); + const [isComplete, setIsComplete] = useState(false); + const [pendingRetryStart, setPendingRetryStart] = useState(false); + const abortRef = useRef(null); + const currentAsyncItemRef = useRef<{ + item: BulkDownloadItem; + password?: string; + skipDecryption: boolean; + } | null>(null); + + const updateItem = useCallback( + (cid: string, updater: (item: BulkDownloadItem) => BulkDownloadItem) => { + setItems((current) => + current.map((item) => (item.cid === cid ? updater(item) : item)), + ); + }, + [], + ); + + useEffect(() => { + if (!isOpen) { + abortRef.current?.abort(); + abortRef.current = null; + currentAsyncItemRef.current = null; + setItems([]); + setIsLoadingMetadata(false); + setHasConfirmedInsecure(false); + setEncryptionChoice(null); + setSharedPassword(''); + setIsRunning(false); + setIsComplete(false); + setPendingRetryStart(false); + return; + } + + let cancelled = false; + setIsLoadingMetadata(true); + setItems(cids.map((cid) => ({ cid, status: 'pending' }))); + + Promise.all( + cids.map(async (cid): Promise => { + try { + const information = + await network.api.fetchUploadedObjectMetadata(cid); + if (isBanned(information.tags)) { + return { + cid, + information, + status: 'skipped', + skippedReason: 'File is banned', + }; + } + if ( + information.status === ObjectStatus.Processing || + information.uploadState.totalNodes === null + ) { + return { + cid, + information, + status: 'skipped', + skippedReason: 'Upload is still processing', + }; + } + return { cid, information, status: 'pending' }; + } catch (error) { + return { + cid, + status: 'failed', + error: + error instanceof Error && error.message + ? error.message + : 'Failed to load metadata', + }; + } + }), + ).then((loadedItems) => { + if (cancelled) return; + setItems(loadedItems); + setIsLoadingMetadata(false); + }); + + return () => { + cancelled = true; + }; + }, [cids, isOpen, network.api]); + + const insecureItems = useMemo( + () => + items.filter( + (item) => + item.status === 'pending' && + item.information && + isInsecure(item.information.tags), + ), + [items], + ); + + const encryptedItemsNeedingDecision = useMemo( + () => + defaultPassword + ? [] + : items.filter( + (item) => item.status === 'pending' && hasEncryption(item), + ), + [defaultPassword, items], + ); + + const canStart = + !isLoadingMetadata && + !isRunning && + items.some(itemIsRunnable) && + (insecureItems.length === 0 || hasConfirmedInsecure) && + (encryptedItemsNeedingDecision.length === 0 || + encryptionChoice === 'download-encrypted' || + encryptionChoice === 'skip' || + (encryptionChoice === 'shared-password' && sharedPassword.length > 0)); + + const encryptionContext = useMemo( + () => ({ defaultPassword, encryptionChoice, sharedPassword }), + [defaultPassword, encryptionChoice, sharedPassword], + ); + + const resolveEncryptionOptions = useCallback( + (item: BulkDownloadItem) => + resolveBulkEncryptionOptions(item, encryptionContext), + [encryptionContext], + ); + + const startQueue = useCallback(async () => { + if (!canStart) return; + + const abortController = new AbortController(); + abortRef.current = abortController; + setIsRunning(true); + setIsComplete(false); + + const runnableItems = items.filter(itemIsRunnable); + for (const item of runnableItems) { + if (abortController.signal.aborted) break; + + if (shouldSkipEncrypted(item, encryptionContext)) { + updateItem(item.cid, (current) => ({ + ...current, + status: 'skipped', + skippedReason: 'Encrypted file skipped', + })); + continue; + } + + // Defense-in-depth: even though `canStart` is gated by + // `hasConfirmedInsecure`, refuse to download insecure files at the + // runtime boundary so future changes to the UI gate can't bypass it. + if (shouldSkipInsecure(item, hasConfirmedInsecure)) { + updateItem(item.cid, (current) => ({ + ...current, + status: 'skipped', + skippedReason: 'Insecure file not confirmed', + })); + continue; + } + + const { password, skipDecryption } = resolveEncryptionOptions(item); + + try { + await runObjectDownloadFlow({ + api: network.api, + downloadService: network.downloadService, + metadata: item.information.metadata, + password, + skipDecryption, + signal: abortController.signal, + onAsyncDownloadsRefresh: updateAsyncDownloads, + getAsyncDownloads: () => + useUserAsyncDownloadsStore.getState().asyncDownloads, + onProgress: (progress) => { + updateItem(item.cid, (current) => ({ + ...current, + progress, + })); + }, + onPhaseChange: (phase) => { + if (phase === 'preparing') { + currentAsyncItemRef.current = { + item, + password, + skipDecryption, + }; + } else if (currentAsyncItemRef.current?.item.cid === item.cid) { + currentAsyncItemRef.current = null; + } + + updateItem(item.cid, (current) => ({ + ...current, + status: phase, + error: undefined, + })); + }, + }); + toast.success(`${shortenString(itemName(item), 30)} downloaded`, { + id: `${toastId}-${item.cid}`, + }); + } catch (error) { + if (currentAsyncItemRef.current?.item.cid === item.cid) { + currentAsyncItemRef.current = null; + } + + if (error instanceof ObjectDownloadAbortedError) { + break; + } + + const errorMessage = + error instanceof InvalidDecryptKey + ? 'Wrong password' + : error instanceof Error && error.message + ? error.message + : 'Download failed'; + updateItem(item.cid, (current) => ({ + ...current, + status: 'failed', + error: errorMessage, + })); + toast.error(`Failed to download ${shortenString(itemName(item), 30)}`, { + id: `${toastId}-${item.cid}`, + }); + } + } + + currentAsyncItemRef.current = null; + setIsRunning(false); + setIsComplete(true); + }, [ + canStart, + encryptionContext, + hasConfirmedInsecure, + items, + network.api, + network.downloadService, + resolveEncryptionOptions, + updateAsyncDownloads, + updateItem, + ]); + + const retryFailed = useCallback(() => { + setItems((current) => + current.map((item) => + item.status === 'failed' && item.information + ? { + ...item, + status: 'pending', + error: undefined, + progress: null, + } + : item, + ), + ); + setIsComplete(false); + setPendingRetryStart(true); + }, []); + + useEffect(() => { + if (pendingRetryStart && canStart) { + setPendingRetryStart(false); + startQueue(); + } + }, [pendingRetryStart, canStart, startQueue]); + + const closeModal = useCallback(() => { + // Only background remaining items when the queue was actually started. + // Without this guard, clicking Cancel before Start Download would + // spuriously kick off async downloads for every selected file. + if (isRunning) { + const currentAsync = currentAsyncItemRef.current; + let backgroundedCount = 0; + + const remainingNotStarted = items + .filter( + (item): item is BulkDownloadItem & { information: ObjectInformation } => + (item.status === 'pending' || item.status === 'checking') && + !!item.information && + item.cid !== currentAsync?.item.cid, + ) + .filter( + (item) => + !shouldSkipEncrypted(item, encryptionContext) && + !shouldSkipInsecure(item, hasConfirmedInsecure), + ); + + if (currentAsync?.item.information) { + addPendingAutoDownload({ + cid: currentAsync.item.information.metadata.dataCid, + password: currentAsync.skipDecryption + ? undefined + : currentAsync.password, + skipDecryption: currentAsync.skipDecryption, + fileName: currentAsync.item.information.metadata.name ?? undefined, + }); + backgroundedCount += 1; + } + + const backgroundPromises = remainingNotStarted.map((item) => { + const { password: itemPassword, skipDecryption } = + resolveBulkEncryptionOptions(item, encryptionContext); + const cid = item.information.metadata.dataCid; + const fileName = item.information.metadata.name ?? undefined; + + return network.api + .createAsyncDownload(cid) + .then(() => { + addPendingAutoDownload({ + cid, + password: skipDecryption ? undefined : itemPassword, + skipDecryption, + fileName, + }); + return true as const; + }) + .catch((err) => { + console.warn( + `[BulkObjectDownloadModal] failed to background ${cid}`, + err, + ); + return false as const; + }); + }); + + if (backgroundedCount > 0 || backgroundPromises.length > 0) { + Promise.all(backgroundPromises).then((results) => { + const totalCount = + backgroundedCount + results.filter(Boolean).length; + if (totalCount > 0) { + updateAsyncDownloads(); + toast.success( + `${totalCount} download${totalCount === 1 ? '' : 's'} will continue in the background. Check Cached Downloads for progress.`, + { id: toastId, duration: 5000 }, + ); + } + }); + } + } + + abortRef.current?.abort(); + if (isComplete) { + onComplete(); + } + onClose(); + }, [ + addPendingAutoDownload, + encryptionContext, + hasConfirmedInsecure, + isComplete, + isRunning, + items, + network.api, + onClose, + onComplete, + updateAsyncDownloads, + ]); + + const handleDialogClose = useCallback(() => { + // Backdrop / Esc close should not abort an in-flight queue — require the + // user to explicitly hit Close while running to background what we can. + if (isRunning) return; + closeModal(); + }, [isRunning, closeModal]); + + const completedCount = items.filter( + (item) => item.status === 'completed', + ).length; + const failedCount = items.filter((item) => item.status === 'failed').length; + const skippedCount = items.filter((item) => item.status === 'skipped').length; + + return ( + + + +
+ + +
+
+ + +
+
+ + Download {cids.length} files + +

+ {completedCount} completed, {failedCount} failed,{' '} + {skippedCount} skipped +

+
+ {isLoadingMetadata && ( +
+ )} +
+ + {insecureItems.length > 0 && !hasConfirmedInsecure && ( +
+ {insecureItems.length} selected file + {insecureItems.length === 1 ? ' is' : 's are'} marked + insecure. + +
+ )} + + {encryptedItemsNeedingDecision.length > 0 && + (insecureItems.length === 0 || hasConfirmedInsecure) && ( +
+

+ {encryptedItemsNeedingDecision.length} encrypted file + {encryptedItemsNeedingDecision.length === 1 + ? '' + : 's'}{' '} + need a bulk decision. +

+
+ + + +
+ {encryptionChoice === 'shared-password' && ( + + setSharedPassword(event.target.value) + } + className='mt-3 block w-full rounded-md border border-gray-300 bg-background p-2 text-foreground shadow-sm' + placeholder='Password' + /> + )} +
+ )} + +
+ {items.map((item) => { + const progress = item.progress?.percentage ?? 0; + return ( +
+
+
+

+ {shortenString(itemName(item), 48)} +

+

+ {item.cid} +

+
+ + {statusLabel[item.status]} + +
+ {item.status === 'downloading' && ( +
+
+
+ )} + {item.progress?.downloadedBytes !== undefined && + item.status === 'downloading' && ( +

+ {formatBytes(item.progress.downloadedBytes)} /{' '} + {formatBytes( + item.progress.totalBytes ?? + Number( + item.information?.metadata.totalSize ?? 0, + ), + )} +

+ )} + {(item.error || item.skippedReason) && ( +

+ {item.error ?? item.skippedReason} +

+ )} +
+ ); + })} +
+ +
+ + {isComplete && failedCount > 0 && !isRunning && ( + + )} + +
+ + +
+
+
+
+ ); +}; diff --git a/apps/frontend/src/components/molecules/ObjectDownloadModal.tsx b/apps/frontend/src/components/molecules/ObjectDownloadModal.tsx index e5e980705..f7c0f4574 100644 --- a/apps/frontend/src/components/molecules/ObjectDownloadModal.tsx +++ b/apps/frontend/src/components/molecules/ObjectDownloadModal.tsx @@ -24,13 +24,14 @@ import { mapObjectInformationFromQueryResult } from 'services/gql/utils'; import { useNetwork } from 'contexts/network'; import { DownloadProgressInfo } from 'services/download'; import { formatBytes } from 'utils/number'; -import { AsyncDownloadStatus, DownloadStatus } from '@auto-drive/models'; -import { getAuthSession } from '@/utils/auth'; import { useUserAsyncDownloadsStore } from '../organisms/UserAsyncDownloads/state'; +import { + ObjectDownloadAbortedError, + ObjectDownloadPhase, + runObjectDownloadFlow, +} from 'services/objectDownloadFlow'; const toastId = 'object-download-modal'; -const MAX_ASYNC_POLL_COUNT = 60; -const ASYNC_POLL_INTERVAL_MS = 10_000; export const ObjectDownloadModal = ({ cid, @@ -51,8 +52,6 @@ export const ObjectDownloadModal = ({ const [downloadError, setDownloadError] = useState(null); const [checkingStatus, setCheckingStatus] = useState(false); const [asyncPreparing, setAsyncPreparing] = useState(false); - const asyncPollRef = useRef | null>(null); - const pollCancelledRef = useRef(false); const defaultPassword = useEncryptionStore((store) => store.password); const network = useNetwork(); const updateAsyncDownloads = useUserAsyncDownloadsStore((e) => e.update); @@ -61,8 +60,8 @@ export const ObjectDownloadModal = ({ ); const downloadInitiatedRef = useRef(null); - const startSyncDownloadRef = useRef<() => Promise>(() => Promise.resolve()); const downloadAbortRef = useRef(null); + const downloadPhaseRef = useRef(null); const handleCloseWhileAsyncPreparing = useCallback(() => { if (!asyncPreparing || !metadata) return; @@ -99,16 +98,12 @@ export const ObjectDownloadModal = ({ setCheckingStatus(false); setAsyncPreparing(false); downloadInitiatedRef.current = null; + downloadPhaseRef.current = null; } return () => { downloadAbortRef.current?.abort(); downloadAbortRef.current = null; - pollCancelledRef.current = true; - if (asyncPollRef.current) { - clearTimeout(asyncPollRef.current); - asyncPollRef.current = null; - } }; }, [cid]); @@ -137,20 +132,48 @@ export const ObjectDownloadModal = ({ }, }); - const startSyncDownload = useCallback(async () => { + const onDownload = useCallback(async () => { if (!metadata) return; - const passwordToUse = skipDecryption ? undefined : password; + + downloadAbortRef.current?.abort(); + const abortController = new AbortController(); + downloadAbortRef.current = abortController; setDownloadError(null); setDownloadProgress(null); + downloadPhaseRef.current = null; try { - await network.downloadService.fetchFile(metadata.dataCid, { - password: passwordToUse, + await runObjectDownloadFlow({ + api: network.api, + downloadService: network.downloadService, + metadata, + password, skipDecryption, + signal: abortController.signal, onProgress: (progress) => { setDownloadProgress(progress); }, + onAsyncDownloadsRefresh: updateAsyncDownloads, + getAsyncDownloads: () => + useUserAsyncDownloadsStore.getState().asyncDownloads, + onPhaseChange: (phase) => { + const previousPhase = downloadPhaseRef.current; + downloadPhaseRef.current = phase; + + setCheckingStatus(phase === 'checking'); + setAsyncPreparing(phase === 'preparing'); + if (phase === 'downloading') { + setIsDownloading(true); + } + + if (phase === 'downloading' && previousPhase === 'preparing') { + toast.success( + `${shortenString(metadata.name ?? 'File', 30)} is ready — downloading now`, + { id: toastId }, + ); + } + }, }); await new Promise((resolve) => setTimeout(resolve, 250)); toast.success( @@ -166,6 +189,8 @@ export const ObjectDownloadModal = ({ setWrongPassword(true); setIsDownloading(false); downloadInitiatedRef.current = null; + } else if (e instanceof ObjectDownloadAbortedError) { + return; } else { console.error('Download failed:', e); const errorMessage = @@ -175,127 +200,19 @@ export const ObjectDownloadModal = ({ setDownloadError(errorMessage); toast.error(errorMessage, { id: toastId }); setIsDownloading(false); + setAsyncPreparing(false); + setCheckingStatus(false); } } - }, [metadata, password, skipDecryption, network.downloadService, onClose]); - - startSyncDownloadRef.current = startSyncDownload; - - const startAsyncDownloadAndPoll = useCallback(async () => { - if (!metadata) return; - - setIsDownloading(false); - setAsyncPreparing(true); - try { - await network.api.createAsyncDownload(metadata.dataCid); - updateAsyncDownloads(); - } catch (e) { - console.error('Failed to create async download:', e); - // Fall back to sync download if async creation fails - setAsyncPreparing(false); - setIsDownloading(true); - startSyncDownloadRef.current(); - return; - } - - pollCancelledRef.current = false; - let pollCount = 0; - const schedulePoll = () => { - asyncPollRef.current = setTimeout(async () => { - if (pollCancelledRef.current) return; - pollCount++; - try { - const status = await network.api.checkDownloadStatus( - metadata.dataCid, - ); - if (pollCancelledRef.current) return; - updateAsyncDownloads(); - if (status === DownloadStatus.Cached) { - asyncPollRef.current = null; - setAsyncPreparing(false); - setIsDownloading(true); - - toast.success( - `${shortenString(metadata.name ?? 'File', 30)} is ready — downloading now`, - { id: toastId }, - ); - startSyncDownloadRef.current(); - return; - } - - const asyncDownloads = - useUserAsyncDownloadsStore.getState().asyncDownloads; - const matchingDownload = asyncDownloads.find( - (d) => d.cid === metadata.dataCid, - ); - if ( - matchingDownload && - (matchingDownload.status === AsyncDownloadStatus.Failed || - matchingDownload.status === AsyncDownloadStatus.Dismissed) - ) { - asyncPollRef.current = null; - setAsyncPreparing(false); - const errorMsg = - matchingDownload.errorMessage || - 'Download failed on the server. Please try again.'; - setDownloadError(errorMsg); - toast.error(errorMsg, { id: toastId }); - return; - } - - if (pollCount >= MAX_ASYNC_POLL_COUNT) { - asyncPollRef.current = null; - setAsyncPreparing(false); - const errorMsg = - 'Download preparation timed out. Please try again later.'; - setDownloadError(errorMsg); - toast.error(errorMsg, { id: toastId }); - return; - } - } catch { - // Ignore transient poll errors - } - if (!pollCancelledRef.current) { - schedulePoll(); - } - }, ASYNC_POLL_INTERVAL_MS); - }; - schedulePoll(); - }, [metadata, network.api, updateAsyncDownloads]); - - const onDownload = useCallback(async () => { - if (!metadata || asyncPreparing) return; - - downloadAbortRef.current?.abort(); - const abortController = new AbortController(); - downloadAbortRef.current = abortController; - const { signal } = abortController; - - setCheckingStatus(true); - - try { - const session = await getAuthSession().catch(() => null); - if (signal.aborted) return; - const hasSession = !!session?.accessToken && !!session?.authProvider; - - if (hasSession) { - const status = await network.api.checkDownloadStatus(metadata.dataCid); - if (signal.aborted) return; - if (status === DownloadStatus.NotCached) { - setCheckingStatus(false); - startAsyncDownloadAndPoll(); - return; - } - } - } catch { - if (signal.aborted) return; - } - - if (signal.aborted) return; - setCheckingStatus(false); - setIsDownloading(true); - startSyncDownload(); - }, [metadata, network.api, startSyncDownload, startAsyncDownloadAndPoll, asyncPreparing]); + }, [ + metadata, + password, + skipDecryption, + network.api, + network.downloadService, + updateAsyncDownloads, + onClose, + ]); const passwordOrNotEncrypted = (metadata && !metadata.uploadOptions?.encryption?.algorithm) || @@ -559,7 +476,8 @@ export const ObjectDownloadModal = ({ ]); // Show modal when there's a view to display OR when downloading/preparing - const shouldShowModal = !!cid && (!!view || isDownloading || asyncPreparing || checkingStatus); + const shouldShowModal = + !!cid && (!!view || isDownloading || asyncPreparing || checkingStatus); if (!shouldShowModal) return <>; @@ -600,7 +518,7 @@ export const ObjectDownloadModal = ({ leaveFrom='opacity-100 scale-100' leaveTo='opacity-0 scale-95' > - + {view ?? progressView} diff --git a/apps/frontend/src/components/organisms/FileTable/index.tsx b/apps/frontend/src/components/organisms/FileTable/index.tsx index de485694c..758f49b33 100644 --- a/apps/frontend/src/components/organisms/FileTable/index.tsx +++ b/apps/frontend/src/components/organisms/FileTable/index.tsx @@ -6,6 +6,7 @@ import { FC, useCallback, useState } from 'react'; import { ObjectShareModal } from '@/components/molecules/ObjectShareModal'; import { ObjectDeleteModal } from '@/components/molecules/ObjectDeleteModal'; import { ObjectDownloadModal } from '@/components/molecules/ObjectDownloadModal'; +import { BulkObjectDownloadModal } from '@/components/molecules/BulkObjectDownloadModal'; import { useUserStore } from 'globalStates/user'; import { Table } from '@/components/molecules/Table'; import { @@ -53,6 +54,7 @@ export const FileTable: FC<{ const [deleteCID, setDeleteCID] = useState(null); const [reportCID, setReportCID] = useState(null); const [selectedFiles, setSelectedFiles] = useState([]); + const [isBulkDownloadOpen, setIsBulkDownloadOpen] = useState(false); const objects = useFileTableState((v) => v.objects); const refetch = useFileTableState((v) => v.fetch); @@ -87,6 +89,12 @@ export const FileTable: FC<{ setShareCID(null)} /> + setIsBulkDownloadOpen(false)} + onComplete={() => setSelectedFiles([])} + />
@@ -120,7 +128,11 @@ export const FileTable: FC<{ {selectedFiles.length} files selected -
diff --git a/apps/frontend/src/services/bulkObjectDownload.ts b/apps/frontend/src/services/bulkObjectDownload.ts new file mode 100644 index 000000000..0caccc6c0 --- /dev/null +++ b/apps/frontend/src/services/bulkObjectDownload.ts @@ -0,0 +1,86 @@ +import { isInsecure, ObjectInformation } from '@auto-drive/models'; +import { DownloadProgressInfo } from 'services/download'; + +export type BulkDownloadStatus = + | 'pending' + | 'skipped' + | 'checking' + | 'preparing' + | 'downloading' + | 'completed' + | 'failed'; + +export type EncryptionChoice = + | 'download-encrypted' + | 'shared-password' + | 'skip'; + +export interface BulkDownloadItem { + cid: string; + information?: ObjectInformation; + status: BulkDownloadStatus; + progress?: DownloadProgressInfo | null; + error?: string; + skippedReason?: string; +} + +export interface EncryptionContext { + defaultPassword?: string | null; + encryptionChoice: EncryptionChoice | null; + sharedPassword: string; +} + +export interface ResolvedEncryptionOptions { + password: string | undefined; + skipDecryption: boolean; +} + +export const hasEncryption = (item: BulkDownloadItem): boolean => + !!item.information?.metadata.uploadOptions?.encryption?.algorithm; + +// Items that are eligible to run in the queue. Insecure items pass this +// predicate but are gated separately at runtime (see `shouldSkipInsecure`) +// so the user's confirmation toggle isn't bypassed. +export const itemIsRunnable = ( + item: BulkDownloadItem, +): item is BulkDownloadItem & { information: ObjectInformation } => + item.status === 'pending' && !!item.information; + +export const shouldSkipInsecure = ( + item: BulkDownloadItem, + hasConfirmedInsecure: boolean, +): boolean => { + if (!item.information) return false; + return isInsecure(item.information.tags) && !hasConfirmedInsecure; +}; + +export const shouldSkipEncrypted = ( + item: BulkDownloadItem, + context: EncryptionContext, +): boolean => + hasEncryption(item) && + !context.defaultPassword && + context.encryptionChoice === 'skip'; + +export const resolveEncryptionOptions = ( + item: BulkDownloadItem, + context: EncryptionContext, +): ResolvedEncryptionOptions => { + if (!hasEncryption(item)) { + return { password: undefined, skipDecryption: false }; + } + + if (context.defaultPassword) { + return { password: context.defaultPassword, skipDecryption: false }; + } + + if (context.encryptionChoice === 'download-encrypted') { + return { password: undefined, skipDecryption: true }; + } + + if (context.encryptionChoice === 'shared-password') { + return { password: context.sharedPassword, skipDecryption: false }; + } + + return { password: undefined, skipDecryption: false }; +}; diff --git a/apps/frontend/src/services/objectDownloadFlow.ts b/apps/frontend/src/services/objectDownloadFlow.ts new file mode 100644 index 000000000..d5b4432e6 --- /dev/null +++ b/apps/frontend/src/services/objectDownloadFlow.ts @@ -0,0 +1,182 @@ +import { OffchainMetadata } from '@autonomys/auto-dag-data'; +import { AsyncDownloadStatus, DownloadStatus } from '@auto-drive/models'; +import { Api } from 'services/api'; +import { DownloadApi, DownloadOptions } from 'services/download'; +import { getAuthSession } from '@/utils/auth'; + +const MAX_ASYNC_POLL_COUNT = 60; +const ASYNC_POLL_INTERVAL_MS = 10_000; + +export type ObjectDownloadPhase = + | 'checking' + | 'preparing' + | 'downloading' + | 'completed'; + +export class ObjectDownloadAbortedError extends Error { + constructor() { + super('Download aborted'); + } +} + +class ObjectDownloadPreparationError extends Error {} + +export interface ObjectDownloadFlowOptions { + api: Api; + downloadService: DownloadApi; + metadata: OffchainMetadata; + password?: string; + skipDecryption?: boolean; + signal?: AbortSignal; + onProgress?: DownloadOptions['onProgress']; + onPhaseChange?: (phase: ObjectDownloadPhase) => void; + onAsyncDownloadsRefresh?: () => void; + getAsyncDownloads?: () => { + cid: string; + status: AsyncDownloadStatus; + errorMessage?: string | null; + }[]; + maxAsyncPollCount?: number; + asyncPollIntervalMs?: number; +} + +const assertNotAborted = (signal?: AbortSignal) => { + if (signal?.aborted) { + throw new ObjectDownloadAbortedError(); + } +}; + +const delay = (ms: number, signal?: AbortSignal) => + new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new ObjectDownloadAbortedError()); + return; + } + + const timeout = setTimeout(resolve, ms); + signal?.addEventListener( + 'abort', + () => { + clearTimeout(timeout); + reject(new ObjectDownloadAbortedError()); + }, + { once: true }, + ); + }); + +export const runObjectDownloadFlow = async ({ + api, + downloadService, + metadata, + password, + skipDecryption = false, + signal, + onProgress, + onPhaseChange, + onAsyncDownloadsRefresh, + getAsyncDownloads, + maxAsyncPollCount = MAX_ASYNC_POLL_COUNT, + asyncPollIntervalMs = ASYNC_POLL_INTERVAL_MS, +}: ObjectDownloadFlowOptions) => { + assertNotAborted(signal); + onPhaseChange?.('checking'); + + let shouldPrepareAsync = false; + try { + const session = await getAuthSession().catch(() => null); + assertNotAborted(signal); + const hasSession = !!session?.accessToken && !!session?.authProvider; + + if (hasSession) { + const status = await api.checkDownloadStatus(metadata.dataCid); + assertNotAborted(signal); + shouldPrepareAsync = status === DownloadStatus.NotCached; + } + } catch (error) { + if (error instanceof ObjectDownloadAbortedError) { + throw error; + } + shouldPrepareAsync = false; + } + + if (shouldPrepareAsync) { + onPhaseChange?.('preparing'); + try { + await api.createAsyncDownload(metadata.dataCid); + onAsyncDownloadsRefresh?.(); + } catch (error) { + if (error instanceof ObjectDownloadAbortedError) { + throw error; + } + shouldPrepareAsync = false; + onPhaseChange?.('checking'); + } + } + + if (shouldPrepareAsync) { + for (let pollCount = 0; pollCount < maxAsyncPollCount; pollCount++) { + await delay(asyncPollIntervalMs, signal); + assertNotAborted(signal); + + let isCached = false; + try { + const status = await api.checkDownloadStatus(metadata.dataCid); + onAsyncDownloadsRefresh?.(); + assertNotAborted(signal); + if (status === DownloadStatus.Cached) { + isCached = true; + } else { + const matchingDownload = getAsyncDownloads?.().find( + (d) => d.cid === metadata.dataCid, + ); + if ( + matchingDownload && + (matchingDownload.status === AsyncDownloadStatus.Failed || + matchingDownload.status === AsyncDownloadStatus.Dismissed) + ) { + throw new ObjectDownloadPreparationError( + matchingDownload.errorMessage || + 'Download failed on the server. Please try again.', + ); + } + } + } catch (error) { + if (error instanceof ObjectDownloadAbortedError) { + throw error; + } + + if (error instanceof ObjectDownloadPreparationError) { + throw error; + } + + // Transient poll-cycle failure (e.g. network blip). Don't fail the + // whole flow — we'll retry next cycle or hit the timeout. Surface + // for debugging so a fully broken gateway doesn't fail silently. + console.warn( + `[objectDownloadFlow] poll cycle ${pollCount + 1}/${maxAsyncPollCount} failed for ${metadata.dataCid}; continuing`, + error, + ); + } + + if (isCached) { + break; + } + + if (pollCount === maxAsyncPollCount - 1) { + throw new ObjectDownloadPreparationError( + 'Download preparation timed out. Please try again later.', + ); + } + } + } + + assertNotAborted(signal); + onPhaseChange?.('downloading'); + await downloadService.fetchFile(metadata.dataCid, { + password: skipDecryption ? undefined : password, + skipDecryption, + onProgress, + }); + assertNotAborted(signal); + onPhaseChange?.('completed'); +};