From feafbe714cae6843791e83bbb3e8d1d217578e97 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Oct 2025 06:21:55 +0000 Subject: [PATCH 1/4] Initial plan From 321db88dd1efe7eb0463e052885ea3e8a790f3dc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Oct 2025 06:32:32 +0000 Subject: [PATCH 2/4] feat: Add search filters for deprecated packages and minimum publish time Co-authored-by: fengmk2 <156269+fengmk2@users.noreply.github.com> --- app/core/service/PackageSearchService.ts | 72 ++++++++++++++++++++++++ app/port/config.ts | 14 +++++ app/repository/PackageRepository.ts | 1 + app/repository/SearchRepository.ts | 1 + config/config.default.ts | 10 ++++ 5 files changed, 98 insertions(+) diff --git a/app/core/service/PackageSearchService.ts b/app/core/service/PackageSearchService.ts index d07d06a99..55fa92d51 100644 --- a/app/core/service/PackageSearchService.ts +++ b/app/core/service/PackageSearchService.ts @@ -124,6 +124,8 @@ export class PackageSearchService extends AbstractService { _npmUser: latestManifest?._npmUser, // 最新版本发布信息 publish_time: latestManifest?.publish_time, + // deprecated message from latest version + deprecated: latestManifest?.deprecated, }; // http://npmmirror.com/package/npm/files/lib/utils/format-search-stream.js#L147-L148 @@ -157,6 +159,7 @@ export class PackageSearchService extends AbstractService { text, scoreEffect: 0.25, }); + const filterQueries = this._buildFilterQueries(); const res = await this.searchRepository.searchPackage({ body: { @@ -169,6 +172,7 @@ export class PackageSearchService extends AbstractService { bool: { should: matchQueries, minimum_should_match: matchQueries.length > 0 ? 1 : 0, + filter: filterQueries, }, }, script_score: scriptScore, @@ -286,6 +290,74 @@ export class PackageSearchService extends AbstractService { ]; } + // oxlint-disable-next-line typescript-eslint/no-explicit-any + private _buildFilterQueries(): any[] { + // oxlint-disable-next-line typescript-eslint/no-explicit-any + const filters: any[] = []; + + // Filter deprecated packages if enabled + if (this.config.cnpmcore.searchFilterDeprecated) { + filters.push({ + bool: { + should: [ + { bool: { must_not: { exists: { field: 'package.deprecated' } } } }, + { term: { 'package.deprecated': '' } }, + ], + }, + }); + } + + // Filter by minimum publish time if configured + const minPublishTime = this.config.cnpmcore.searchMinPublishTime; + if (minPublishTime) { + const minDate = this._parseMinPublishTime(minPublishTime); + if (minDate) { + filters.push({ + range: { + 'package.date': { + lte: minDate.toISOString(), + }, + }, + }); + } + } + + return filters; + } + + private _parseMinPublishTime(timeStr: string): Date | null { + const match = timeStr.match(/^(\d+)([hdw])$/); + if (!match) { + this.logger.warn( + '[PackageSearchService._parseMinPublishTime] Invalid format: %s, expected format like "2w", "1d", "24h"', + timeStr + ); + return null; + } + + const value = parseInt(match[1], 10); + const unit = match[2]; + + const now = dayjs(); + let minDate: dayjs.Dayjs; + + switch (unit) { + case 'h': + minDate = now.subtract(value, 'hour'); + break; + case 'd': + minDate = now.subtract(value, 'day'); + break; + case 'w': + minDate = now.subtract(value, 'week'); + break; + default: + return null; + } + + return minDate.toDate(); + } + private _buildScriptScore(params: { text: string | undefined; scoreEffect: number; diff --git a/app/port/config.ts b/app/port/config.ts index 6ff4270e6..725f188f5 100644 --- a/app/port/config.ts +++ b/app/port/config.ts @@ -181,6 +181,20 @@ export interface CnpmcoreConfig { */ strictValidatePackageDeps?: boolean; + /** + * Filter deprecated packages from search results + * Default: true (filter out deprecated packages) + */ + searchFilterDeprecated?: boolean; + + /** + * Minimum publish time before packages appear in search results + * Format: number + unit (h=hours, d=days, w=weeks) + * Examples: '2w' (2 weeks), '1d' (1 day), '24h' (24 hours) + * Default: '' (no filtering) + */ + searchMinPublishTime?: string; + /** * database config */ diff --git a/app/repository/PackageRepository.ts b/app/repository/PackageRepository.ts index b33de9dc3..88e924fa9 100644 --- a/app/repository/PackageRepository.ts +++ b/app/repository/PackageRepository.ts @@ -45,6 +45,7 @@ export type AbbreviatedPackageManifestType = Pick< export type PackageJSONType = CnpmcorePatchInfo & { name: string; version: string; + deprecated?: string; readme?: string; description?: string; keywords?: string[]; diff --git a/app/repository/SearchRepository.ts b/app/repository/SearchRepository.ts index 9a7d1717c..d9ea55074 100644 --- a/app/repository/SearchRepository.ts +++ b/app/repository/SearchRepository.ts @@ -27,6 +27,7 @@ export type SearchMappingType = Pick & created: Date; modified: Date; author?: AuthorType | undefined; + deprecated?: string; _npmUser?: { name: string; email: string; diff --git a/config/config.default.ts b/config/config.default.ts index 7cc2ad896..f04035bfd 100644 --- a/config/config.default.ts +++ b/config/config.default.ts @@ -74,6 +74,16 @@ export const cnpmcoreConfig: CnpmcoreConfig = { elasticsearchIndex: 'cnpmcore_packages', strictValidateTarballPkg: false, strictValidatePackageDeps: false, + searchFilterDeprecated: env( + 'CNPMCORE_CONFIG_SEARCH_FILTER_DEPRECATED', + 'boolean', + true + ), + searchMinPublishTime: env( + 'CNPMCORE_CONFIG_SEARCH_MIN_PUBLISH_TIME', + 'string', + '' + ), database: { type: database.type, }, From c7abcc7e9dfcfc9721095f299d6194e8013fbfdd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Oct 2025 06:38:32 +0000 Subject: [PATCH 3/4] test: Add comprehensive tests for search filtering features Co-authored-by: fengmk2 <156269+fengmk2@users.noreply.github.com> --- test/TestUtil.ts | 4 + .../package/SearchPackageController.test.ts | 179 ++++++++++++++++++ 2 files changed, 183 insertions(+) diff --git a/test/TestUtil.ts b/test/TestUtil.ts index cc2607aef..127bbce9f 100644 --- a/test/TestUtil.ts +++ b/test/TestUtil.ts @@ -35,6 +35,7 @@ interface PackageOptions { description?: string; registryId?: string; main?: string; + deprecated?: string; } interface UserOptions { @@ -260,6 +261,9 @@ export class TestUtil { if ('main' in options) { version.main = options.main; } + if (options.deprecated) { + version.deprecated = options.deprecated; + } } return pkg; } diff --git a/test/port/controller/package/SearchPackageController.test.ts b/test/port/controller/package/SearchPackageController.test.ts index cd2ff09a0..21fa06777 100644 --- a/test/port/controller/package/SearchPackageController.test.ts +++ b/test/port/controller/package/SearchPackageController.test.ts @@ -93,6 +93,141 @@ describe('test/port/controller/package/SearchPackageController.test.ts', () => { assert.equal(res.body.objects[0].package.name, 'example'); assert.equal(res.body.total, 1); }); + + it('should filter deprecated packages by default', async () => { + let capturedQuery: any; + mockES.add( + { + method: 'POST', + path: `/${app.config.cnpmcore.elasticsearchIndex}/_search`, + }, + (params: any) => { + capturedQuery = params.body; + return { + hits: { + total: { value: 0, relation: 'eq' }, + hits: [], + }, + }; + } + ); + + await app.httpRequest().get('/-/v1/search?text=example&from=0&size=1'); + + // Verify that filter for deprecated packages is added + assert.ok(capturedQuery); + assert.ok(capturedQuery.query.function_score.query.bool.filter); + const filters = capturedQuery.query.function_score.query.bool.filter; + assert.ok(filters.length > 0); + + // Find the deprecated filter + const deprecatedFilter = filters.find((f: any) => + f.bool && f.bool.should && + f.bool.should.some((s: any) => + s.bool?.must_not?.exists?.field === 'package.deprecated' + ) + ); + assert.ok(deprecatedFilter, 'Should have deprecated filter'); + }); + + it('should not filter deprecated packages when searchFilterDeprecated is false', async () => { + mock(app.config.cnpmcore, 'searchFilterDeprecated', false); + let capturedQuery: any; + mockES.add( + { + method: 'POST', + path: `/${app.config.cnpmcore.elasticsearchIndex}/_search`, + }, + (params: any) => { + capturedQuery = params.body; + return { + hits: { + total: { value: 0, relation: 'eq' }, + hits: [], + }, + }; + } + ); + + await app.httpRequest().get('/-/v1/search?text=example&from=0&size=1'); + + // Verify no deprecated filter is added + const filters = capturedQuery?.query?.function_score?.query?.bool?.filter || []; + const deprecatedFilter = filters.find((f: any) => + f.bool && f.bool.should && + f.bool.should.some((s: any) => + s.bool?.must_not?.exists?.field === 'package.deprecated' + ) + ); + assert.ok(!deprecatedFilter, 'Should not have deprecated filter'); + }); + + it('should filter packages by minimum publish time', async () => { + mock(app.config.cnpmcore, 'searchMinPublishTime', '2w'); + let capturedQuery: any; + mockES.add( + { + method: 'POST', + path: `/${app.config.cnpmcore.elasticsearchIndex}/_search`, + }, + (params: any) => { + capturedQuery = params.body; + return { + hits: { + total: { value: 0, relation: 'eq' }, + hits: [], + }, + }; + } + ); + + await app.httpRequest().get('/-/v1/search?text=example&from=0&size=1'); + + // Verify that time filter is added + const filters = capturedQuery?.query?.function_score?.query?.bool?.filter || []; + const timeFilter = filters.find((f: any) => f.range && f.range['package.date']); + assert.ok(timeFilter, 'Should have time filter'); + assert.ok(timeFilter.range['package.date'].lte, 'Should have lte constraint'); + }); + + it('should accept different time formats for searchMinPublishTime', async () => { + const testCases = [ + { input: '24h', expectedUnit: 'hour' }, + { input: '7d', expectedUnit: 'day' }, + { input: '2w', expectedUnit: 'week' }, + ]; + + for (const testCase of testCases) { + mock(app.config.cnpmcore, 'searchMinPublishTime', testCase.input); + let capturedQuery: any; + mockES.add( + { + method: 'POST', + path: `/${app.config.cnpmcore.elasticsearchIndex}/_search`, + }, + (params: any) => { + capturedQuery = params.body; + return { + hits: { + total: { value: 0, relation: 'eq' }, + hits: [], + }, + }; + } + ); + + await app.httpRequest().get('/-/v1/search?text=example&from=0&size=1'); + + const filters = capturedQuery?.query?.function_score?.query?.bool?.filter || []; + const timeFilter = filters.find((f: any) => f.range && f.range['package.date']); + assert.ok(timeFilter, `Should have time filter for ${testCase.input}`); + + mockES.clearAll(); + mock.restore(); + mock(app.config.cnpmcore, 'enableElasticsearch', true); + mock(app.config.cnpmcore, 'elasticsearchIndex', 'cnpmcore_packages'); + } + }); }); describe('[PUT /-/v1/search/sync/:fullname] sync()', async () => { @@ -130,6 +265,50 @@ describe('test/port/controller/package/SearchPackageController.test.ts', () => { res = await app.httpRequest().put(`/-/v1/search/sync/${name}`); assert.equal(res.body.package, name); }); + + it('should upsert a deprecated package with deprecated field', async () => { + const name = 'testmodule-deprecated-package'; + let capturedDocument: any; + mockES.add( + { + method: 'PUT', + path: `/${app.config.cnpmcore.elasticsearchIndex}/_doc/:id`, + }, + (params: any) => { + capturedDocument = params.document; + return { + _id: name, + }; + } + ); + mock(app.config.cnpmcore, 'allowPublishNonScopePackage', true); + mock(app.config.cnpmcore, 'registry', 'https://registry.example.com'); + + const pkg = await TestUtil.getFullPackage({ + name, + version: '1.0.0', + deprecated: 'This package is deprecated, use new-package instead' + }); + + let res = await app + .httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + assert.equal(res.body.ok, true); + + res = await app.httpRequest().put(`/-/v1/search/sync/${name}`); + assert.equal(res.body.package, name); + + // Verify the deprecated field was synced to Elasticsearch + assert.ok(capturedDocument); + assert.equal( + capturedDocument.package.deprecated, + 'This package is deprecated, use new-package instead' + ); + }); }); describe('[DELETE /-/v1/search/sync/:fullname] delete()', async () => { From d97f2816f75fcd49a668b2642958827203409a49 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Oct 2025 06:39:53 +0000 Subject: [PATCH 4/4] docs: Add environment variable documentation for search filters Co-authored-by: fengmk2 <156269+fengmk2@users.noreply.github.com> --- .env.example | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.env.example b/.env.example index 22aa06195..82cd3df57 100644 --- a/.env.example +++ b/.env.example @@ -13,6 +13,15 @@ # CNPMCORE_CONFIG_ES_CLIENT_AUTH_USERNAME=elastic # CNPMCORE_CONFIG_ES_CLIENT_AUTH_PASSWORD=abcdef +# Search filter configuration (requires Elasticsearch) +# Filter deprecated packages from search results (default: true) +# CNPMCORE_CONFIG_SEARCH_FILTER_DEPRECATED=true +# Minimum publish time before packages appear in search results +# Format: number + unit (h=hours, d=days, w=weeks) +# Examples: "2w" (2 weeks), "1d" (1 day), "24h" (24 hours) +# Default: empty string (no filtering) +# CNPMCORE_CONFIG_SEARCH_MIN_PUBLISH_TIME=2w + # https://github.com/cnpm/cnpmcore/blob/next/docs/elasticsearch-setup.md#%E6%96%B0%E5%BB%BA-env-%E6%96%87%E4%BB%B6 # Password for the 'elastic' user (at least 6 characters) ELASTIC_PASSWORD="abcdef"