diff --git a/app/core/service/PackageManagerService.ts b/app/core/service/PackageManagerService.ts index 323ccb24d..8972db0ab 100644 --- a/app/core/service/PackageManagerService.ts +++ b/app/core/service/PackageManagerService.ts @@ -30,6 +30,7 @@ import type { PackageManifestType, PackageRepository, } from '../../repository/PackageRepository.js'; +import type { PackageVersionFileRepository } from '../../repository/PackageVersionFileRepository.js'; import type { PackageVersionBlockRepository } from '../../repository/PackageVersionBlockRepository.js'; import type { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository.js'; import type { DistRepository } from '../../repository/DistRepository.js'; @@ -101,6 +102,8 @@ export class PackageManagerService extends AbstractService { @Inject() private readonly packageRepository: PackageRepository; @Inject() + private readonly packageVersionFileRepository: PackageVersionFileRepository; + @Inject() private readonly packageVersionBlockRepository: PackageVersionBlockRepository; @Inject() private readonly packageVersionDownloadRepository: PackageVersionDownloadRepository; @@ -726,6 +729,40 @@ export class PackageManagerService extends AbstractService { ]); // remove from repository await this.packageRepository.removePackageVersion(pkgVersion); + + // remove package version files + await this.removePackageVersionFileAndDist(pkgVersion); + } + + public async removePackageVersionFileAndDist(pkgVersion: PackageVersion) { + let fileDists: Dist[] = []; + if (!this.config.cnpmcore.enableUnpkg) return fileDists; + if (!this.config.cnpmcore.enableSyncUnpkgFiles) return fileDists; + + fileDists = + await this.packageVersionFileRepository.findPackageVersionFileDists( + pkgVersion.packageVersionId + ); + + // remove nfs dists + await pMap( + fileDists, + async dist => { + await this.distRepository.destroyDist(dist); + }, + { + concurrency: 50, + stopOnError: false, + } + ); + + // remove package version files from repository + await this.packageVersionFileRepository.removePackageVersionFiles( + pkgVersion.packageVersionId, + fileDists.map(dists => dists.distId) + ); + + return fileDists; } public async unpublishPackage(pkg: Package) { diff --git a/app/core/service/PackageVersionFileService.ts b/app/core/service/PackageVersionFileService.ts index 891914648..c3ea0f0c5 100644 --- a/app/core/service/PackageVersionFileService.ts +++ b/app/core/service/PackageVersionFileService.ts @@ -335,6 +335,33 @@ export class PackageVersionFileService extends AbstractService { } } + async removePackageVersionFiles(pkgVersion: PackageVersion) { + const files: PackageVersionFile[] = []; + if (!this.config.cnpmcore.enableUnpkg) return files; + if (!this.config.cnpmcore.enableSyncUnpkgFiles) return files; + const pkg = await this.packageRepository.findPackageByPackageId( + pkgVersion.packageId + ); + if (!pkg) return files; + + const fileDists = + await this.packageManagerService.removePackageVersionFileAndDist( + pkgVersion + ); + + return fileDists.map(dist => { + const { directory, name } = this.#getDirectoryAndName(dist.path); + return PackageVersionFile.create({ + packageVersionId: pkgVersion.packageVersionId, + directory, + name, + dist, + contentType: mimeLookup(dist.path), + mtime: pkgVersion.publishTime, + }); + }); + } + async #savePackageVersionFile( pkg: Package, pkgVersion: PackageVersion, diff --git a/app/port/controller/PackageVersionFileController.ts b/app/port/controller/PackageVersionFileController.ts index 9bbf701ad..89b93abd3 100644 --- a/app/port/controller/PackageVersionFileController.ts +++ b/app/port/controller/PackageVersionFileController.ts @@ -100,6 +100,36 @@ export class PackageVersionFileController extends AbstractController { return files.map(file => formatFileItem(file)); } + @HTTPMethod({ + // DELETE /:fullname/:versionSpec/files + path: `/:fullname(${FULLNAME_REG_STRING})/:versionSpec/files`, + method: HTTPMethodEnum.DELETE, + }) + @Middleware(AdminAccess) + async removeFiles( + @Context() ctx: EggContext, + @HTTPParam() fullname: string, + @HTTPParam() versionSpec: string + ) { + ctx.tValidate(Spec, `${fullname}@${versionSpec}`); + this.#requireUnpkgEnable(); + const [scope, name] = getScopeAndName(fullname); + const { packageVersion } = + await this.packageManagerService.showPackageVersionByVersionOrTag( + scope, + name, + versionSpec + ); + if (!packageVersion) { + throw new NotFoundError(`${fullname}@${versionSpec} not found`); + } + const files = + await this.packageVersionFileService.removePackageVersionFiles( + packageVersion + ); + return files.map(file => formatFileItem(file)); + } + @HTTPMethod({ // GET /:fullname/:versionSpec/files => /:fullname/:versionSpec/files/${pkg.main} // GET /:fullname/:versionSpec/files?meta diff --git a/app/repository/PackageVersionFileRepository.ts b/app/repository/PackageVersionFileRepository.ts index bb80ac3ce..c954741b8 100644 --- a/app/repository/PackageVersionFileRepository.ts +++ b/app/repository/PackageVersionFileRepository.ts @@ -50,6 +50,24 @@ export class PackageVersionFileRepository extends AbstractRepository { ); } + async findPackageVersionFileDists(packageVersionId: string) { + const fileDists: DistEntity[] = []; + const queue = ['/']; + while (queue.length > 0) { + const directory = queue.shift(); + if (!directory) { + continue; + } + const { files, directories } = await this.listPackageVersionFiles( + packageVersionId, + directory + ); + fileDists.push(...files.map(file => file.dist)); + queue.push(...directories); + } + return fileDists; + } + async listPackageVersionFiles(packageVersionId: string, directory: string) { const isRoot = directory === '/'; const where = isRoot @@ -96,6 +114,47 @@ export class PackageVersionFileRepository extends AbstractRepository { return { files, directories: Array.from(subDirectories) }; } + async removePackageVersionFiles( + packageVersionId: string, + distIds?: string[] + ) { + if (!distIds) { + const fileDists = + await this.findPackageVersionFileDists(packageVersionId); + distIds = fileDists.map(dist => dist.distId); + } + + await this.PackageVersionFile.transaction(async transaction => { + const removeCount = await this.PackageVersionFile.remove( + { packageVersionId }, + true, + transaction + ); + this.logger.info( + '[PackageVersionFileRepository:removePackageVersionFiles:remove] %d rows in PackageVersionFile, packageVersionId: %s', + removeCount, + packageVersionId + ); + + if (distIds.length > 0) { + const distCount = await this.Dist.remove( + { + distId: distIds, + }, + true, + transaction + ); + this.logger.info( + '[PackageVersionFileRepository:removePackageVersionFiles:remove] %d rows in Dist, packageVersionId: %s', + distCount, + packageVersionId + ); + } + }); + + return distIds; + } + async hasPackageVersionFiles(packageVersionId: string) { const model = await this.PackageVersionFile.findOne({ packageVersionId }); return !!model; diff --git a/test/port/controller/PackageVersionFileController/removeFiles.test.ts b/test/port/controller/PackageVersionFileController/removeFiles.test.ts new file mode 100644 index 000000000..90059b329 --- /dev/null +++ b/test/port/controller/PackageVersionFileController/removeFiles.test.ts @@ -0,0 +1,145 @@ +import assert from 'node:assert/strict'; +import { app, mock } from '@eggjs/mock/bootstrap'; + +import { TestUtil, type TestUser } from '../../../../test/TestUtil.js'; + +describe('test/port/controller/PackageVersionFileController/removeFiles.test.ts', () => { + let publisher: TestUser; + let adminUser: TestUser; + beforeEach(async () => { + publisher = await TestUtil.createUser(); + adminUser = await TestUtil.createAdmin(); + }); + + describe('[DELETE /:fullname/:versionSpec/files] removeFiles()', () => { + it('should 404 when enableUnpkg = false', async () => { + mock(app.config.cnpmcore, 'allowPublishNonScopePackage', true); + mock(app.config.cnpmcore, 'enableUnpkg', false); + const pkg = await TestUtil.getFullPackage({ + name: 'foo', + version: '1.0.0', + versionObject: { + description: 'work with utf8mb4 ๐Ÿ’ฉ, ๐Œ† utf8_unicode_ci, foo๐Œ†bar ๐Ÿป', + }, + }); + await app + .httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + const res = await app + .httpRequest() + .delete('/foo/1.0.0/files') + .set('authorization', adminUser.authorization) + .expect(404) + .expect('content-type', 'application/json; charset=utf-8'); + assert.equal(res.body.error, '[NOT_FOUND] Not Found'); + }); + + it('should work', async () => { + mock(app.config.cnpmcore, 'allowPublishNonScopePackage', true); + mock(app.config.cnpmcore, 'enableUnpkg', true); + mock(app.config.cnpmcore, 'enableSyncUnpkgFiles', true); + app.mockLog(); + + const pkg = await TestUtil.getFullPackage({ + name: 'foo', + version: '1.0.0', + versionObject: { + description: 'work with utf8mb4 ๐Ÿ’ฉ, ๐Œ† utf8_unicode_ci, foo๐Œ†bar ๐Ÿป', + }, + }); + + // publish package + await app + .httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + const pkgResponse = await app + .httpRequest() + .get(`/${pkg.name}/1.0.0`) + .expect(200); + const publishTime = new Date(pkgResponse.body.publish_time).toISOString(); + + // sync package version files + await app + .httpRequest() + .put(`/${pkg.name}/1.0.0/files`) + .set('authorization', adminUser.authorization) + .expect(200) + .expect('content-type', 'application/json; charset=utf-8'); + await app + .httpRequest() + .get(`/${pkg.name}/1.0.0/files/package.json`) + .expect(200); + + // remove package version files + const packageVersionFilesDeleteResponse = await app + .httpRequest() + .delete(`/${pkg.name}/1.0.0/files`) + .set('authorization', adminUser.authorization) + .expect(200) + .expect('content-type', 'application/json; charset=utf-8'); + assert.deepEqual(packageVersionFilesDeleteResponse.body, [ + { + path: `/packages/${pkg.name}/1.0.0/files/package.json`, + type: 'file', + contentType: 'application/json', + integrity: + 'sha512-yTg/L7tUtFK54aNH3iwgIp7sF3PiAcUrIEUo06bSNq3haIKRnagy6qOwxiEmtfAtNarbjmEpl31ZymySsECi3Q==', + lastModified: publishTime, + size: 209, + }, + ]); + app.expectLog( + `DELETE /${pkg.name}/1.0.0/files] [nfsAdapter:remove] key: /packages/foo/1.0.0/files/package.json` + ); + app.expectLog( + `DELETE /${pkg.name}/1.0.0/files] [PackageVersionFileRepository:removePackageVersionFiles:remove] 1 rows in PackageVersionFile` + ); + app.expectLog( + `DELETE /${pkg.name}/1.0.0/files] [PackageVersionFileRepository:removePackageVersionFiles:remove] 1 rows in Dist` + ); + + // remove again + const packageVersionFilesDeleteResponse2 = await app + .httpRequest() + .delete(`/${pkg.name}/1.0.0/files`) + .set('authorization', adminUser.authorization) + .expect(200) + .expect('content-type', 'application/json; charset=utf-8'); + assert.deepEqual(packageVersionFilesDeleteResponse2.body, []); + app.expectLog( + `DELETE /${pkg.name}/1.0.0/files] [PackageVersionFileRepository:removePackageVersionFiles:remove] 0 rows in PackageVersionFile` + ); + app.notExpectLog( + `DELETE /${pkg.name}/1.0.0/files] [PackageVersionFileRepository:removePackageVersionFiles:remove] 0 rows in Dist` + ); + }); + + it('should 404 when package not exists', async () => { + const res = await app + .httpRequest() + .delete('/@cnpm/foonot-exists/1.0.40000404/files') + .set('authorization', adminUser.authorization) + .expect(404); + assert.equal( + res.body.error, + '[NOT_FOUND] @cnpm/foonot-exists@1.0.40000404 not found' + ); + }); + + it('should 403 when non-admin request', async () => { + const res = await app + .httpRequest() + .delete('/@cnpm/foonot-exists/1.0.40000404/files') + .expect(403); + assert.equal(res.body.error, '[FORBIDDEN] Not allow to access'); + }); + }); +}); diff --git a/test/port/controller/package/RemovePackageVersionController.test.ts b/test/port/controller/package/RemovePackageVersionController.test.ts index e7953560f..ed9c6271d 100644 --- a/test/port/controller/package/RemovePackageVersionController.test.ts +++ b/test/port/controller/package/RemovePackageVersionController.test.ts @@ -208,6 +208,69 @@ describe('test/port/controller/package/RemovePackageVersionController.test.ts', assert.ok(res.body['dist-tags'].latest === '2.0.0'); }); + it('should remove package version files', async () => { + app.mockLog(); + mock(app.config.cnpmcore, 'allowPublishNonScopePackage', true); + mock(app.config.cnpmcore, 'enableUnpkg', true); + mock(app.config.cnpmcore, 'enableSyncUnpkgFiles', true); + const { pkg } = await TestUtil.createPackage({ + name: 'foo', + version: '1.0.0', + isPrivate: false, + }); + let res = await app.httpRequest().get(`/${pkg.name}`).expect(200); + const adminUser = await TestUtil.createUser({ name: 'cnpmcore_admin' }); + const pkgVersion = res.body; + + await app + .httpRequest() + .get(`/${pkg.name}/1.0.0/files/package.json`) + .expect(200); + + res = await app + .httpRequest() + .delete(`/${pkg.name}/-rev/${pkgVersion._rev}`) + .set('authorization', adminUser.authorization) + .set('npm-command', 'unpublish') + .set('user-agent', adminUser.ua) + .expect(200); + assert.equal(res.body.ok, true); + + app.expectLog( + `DELETE /${pkg.name}/-rev/${pkgVersion._rev}] [PackageController:removeVersion] ${pkg.name}@1.0.0` + ); + app.expectLog( + `DELETE /${pkg.name}/-rev/${pkgVersion._rev}] [nfsAdapter:remove] key: /packages/${pkg.name}/1.0.0/abbreviated.json` + ); + app.expectLog( + `DELETE /${pkg.name}/-rev/${pkgVersion._rev}] [nfsAdapter:remove] key: /packages/${pkg.name}/1.0.0/package.json` + ); + app.expectLog( + `DELETE /${pkg.name}/-rev/${pkgVersion._rev}] [nfsAdapter:remove] key: /packages/${pkg.name}/1.0.0/readme.md` + ); + app.expectLog( + `DELETE /${pkg.name}/-rev/${pkgVersion._rev}] [nfsAdapter:remove] key: /packages/${pkg.name}/1.0.0/${pkg.name}-1.0.0.tgz` + ); + app.expectLog( + `DELETE /${pkg.name}/-rev/${pkgVersion._rev}] [PackageRepository:removePackageVersion:remove] 4 dist rows, 1 rows` + ); + + app.expectLog( + `DELETE /${pkg.name}/-rev/${pkgVersion._rev}] [nfsAdapter:remove] key: /packages/${pkg.name}/1.0.0/files/package.json` + ); + app.expectLog( + `DELETE /${pkg.name}/-rev/${pkgVersion._rev}] [PackageVersionFileRepository:removePackageVersionFiles:remove] 1 rows in PackageVersionFile` + ); + app.expectLog( + `DELETE /${pkg.name}/-rev/${pkgVersion._rev}] [PackageVersionFileRepository:removePackageVersionFiles:remove] 1 rows in Dist` + ); + + await app + .httpRequest() + .get(`/${pkg.name}/1.0.0/files/package.json`) + .expect(404); + }); + it('should 404 when version not exists', async () => { const pkg = await TestUtil.getFullPackage({ name: '@cnpm/foo',