From 5a7e78601f347066c283ac52fe58def601b75a4d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Oct 2025 06:38:13 +0000 Subject: [PATCH] feat: add search filters for deprecated packages and newly published packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implemented two new search filter configurations: 1. `searchExcludeDeprecated` - Filter deprecated packages from search results - Default: true (deprecated packages are excluded by default) - Can be disabled by setting to false - Controlled by env: CNPMCORE_CONFIG_SEARCH_EXCLUDE_DEPRECATED 2. `searchPackageMinAge` - Filter newly published packages - Default: empty string (no filter applied) - Supports time format: '2w' (weeks), '14d' (days), '336h' (hours) - Controlled by env: CNPMCORE_CONFIG_SEARCH_PACKAGE_MIN_AGE Changes: - Added `deprecated` field to SearchMappingType and sync it to Elasticsearch - Added `_buildFilterQueries()` method to build Elasticsearch filter conditions - Added `_parseTimeString()` helper to parse time strings (h/d/w format) - Added comprehensive unit tests for the new filter functionality Closes #858 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- app/core/service/PackageSearchService.ts | 74 ++++++++ app/port/config.ts | 14 ++ app/repository/SearchRepository.ts | 3 +- config/config.default.ts | 2 + .../package/SearchPackageController.test.ts | 160 ++++++++++++++++++ 5 files changed, 252 insertions(+), 1 deletion(-) diff --git a/app/core/service/PackageSearchService.ts b/app/core/service/PackageSearchService.ts index d07d06a99..88db0d954 100644 --- a/app/core/service/PackageSearchService.ts +++ b/app/core/service/PackageSearchService.ts @@ -124,6 +124,8 @@ export class PackageSearchService extends AbstractService { _npmUser: latestManifest?._npmUser, // 最新版本发布信息 publish_time: latestManifest?.publish_time, + // 最新版本的 deprecated 信息 + deprecated: latestManifest?.deprecated as string | undefined, }; // http://npmmirror.com/package/npm/files/lib/utils/format-search-stream.js#L147-L148 @@ -157,6 +159,7 @@ export class PackageSearchService extends AbstractService { text, scoreEffect: 0.25, }); + const filterQueries = this._buildFilterQueries(); const res = await this.searchRepository.searchPackage({ body: { @@ -169,6 +172,7 @@ export class PackageSearchService extends AbstractService { bool: { should: matchQueries, minimum_should_match: matchQueries.length > 0 ? 1 : 0, + filter: filterQueries, }, }, script_score: scriptScore, @@ -303,4 +307,74 @@ export class PackageSearchService extends AbstractService { }, }; } + + private _buildFilterQueries() { + // oxlint-disable-next-line typescript-eslint/no-explicit-any + const filters: any[] = []; + + // Filter deprecated packages + const excludeDeprecated = this.config.cnpmcore.searchExcludeDeprecated ?? true; + if (excludeDeprecated) { + filters.push({ + bool: { + must_not: { + exists: { + field: 'package.deprecated', + }, + }, + }, + }); + } + + // Filter newly published packages + const minAge = this.config.cnpmcore.searchPackageMinAge; + if (minAge) { + const minAgeMs = this._parseTimeString(minAge); + if (minAgeMs > 0) { + const thresholdDate = new Date(Date.now() - minAgeMs); + filters.push({ + range: { + 'package.date': { + lte: thresholdDate.toISOString(), + }, + }, + }); + } + } + + return filters; + } + + /** + * Parse time string to milliseconds + * Supports formats: '2w' (weeks), '14d' (days), '336h' (hours) + * @param timeStr - time string like '2w', '14d', '336h' + * @returns milliseconds, or 0 if invalid + */ + private _parseTimeString(timeStr: string): number { + if (!timeStr) return 0; + + const match = timeStr.match(/^(\d+)([hdw])$/); + if (!match) { + this.logger.warn( + '[PackageSearchService._parseTimeString] Invalid time format: %s, expected format like "2w", "14d", or "336h"', + timeStr + ); + return 0; + } + + const value = parseInt(match[1], 10); + const unit = match[2]; + + switch (unit) { + case 'h': // hours + return value * 60 * 60 * 1000; + case 'd': // days + return value * 24 * 60 * 60 * 1000; + case 'w': // weeks + return value * 7 * 24 * 60 * 60 * 1000; + default: + return 0; + } + } } diff --git a/app/port/config.ts b/app/port/config.ts index 6ff4270e6..39b8bc1b0 100644 --- a/app/port/config.ts +++ b/app/port/config.ts @@ -187,4 +187,18 @@ export interface CnpmcoreConfig { database: { type: DATABASE_TYPE | string; }; + + /** + * search package minimum age filter + * packages published within this time range will be excluded from search results + * support format: '2w' (weeks), '14d' (days), '336h' (hours) + * default is empty string (no filter) + */ + searchPackageMinAge?: string; + + /** + * exclude deprecated packages from search results + * default is true + */ + searchExcludeDeprecated?: boolean; } diff --git a/app/repository/SearchRepository.ts b/app/repository/SearchRepository.ts index 9a7d1717c..9b8d0156e 100644 --- a/app/repository/SearchRepository.ts +++ b/app/repository/SearchRepository.ts @@ -16,7 +16,8 @@ export type SearchJSONPickKey = | 'license' | 'maintainers' | 'dist-tags' - | '_source_registry_name'; + | '_source_registry_name' + | 'deprecated'; export type SearchMappingType = Pick & CnpmcorePatchInfo & { diff --git a/config/config.default.ts b/config/config.default.ts index 7cc2ad896..5fad13788 100644 --- a/config/config.default.ts +++ b/config/config.default.ts @@ -77,6 +77,8 @@ export const cnpmcoreConfig: CnpmcoreConfig = { database: { type: database.type, }, + searchPackageMinAge: env('CNPMCORE_CONFIG_SEARCH_PACKAGE_MIN_AGE', 'string', ''), + searchExcludeDeprecated: env('CNPMCORE_CONFIG_SEARCH_EXCLUDE_DEPRECATED', 'boolean', true), }; export interface NFSConfig { diff --git a/test/port/controller/package/SearchPackageController.test.ts b/test/port/controller/package/SearchPackageController.test.ts index cd2ff09a0..dc849c7f4 100644 --- a/test/port/controller/package/SearchPackageController.test.ts +++ b/test/port/controller/package/SearchPackageController.test.ts @@ -93,6 +93,166 @@ describe('test/port/controller/package/SearchPackageController.test.ts', () => { assert.equal(res.body.objects[0].package.name, 'example'); assert.equal(res.body.total, 1); }); + + it('should filter deprecated packages by default', async () => { + let capturedQuery: any; + mockES.add( + { + method: 'POST', + path: `/${app.config.cnpmcore.elasticsearchIndex}/_search`, + }, + (params: any) => { + capturedQuery = params.body; + return { + hits: { + total: { value: 0, relation: 'eq' }, + hits: [], + }, + }; + } + ); + await app + .httpRequest() + .get('/-/v1/search?text=example&from=0&size=1') + .expect(200); + + // Verify that the query includes deprecated filter + assert(capturedQuery); + assert(capturedQuery.query.function_score.query.bool.filter); + const filters = capturedQuery.query.function_score.query.bool.filter; + const deprecatedFilter = filters.find((f: any) => + f.bool?.must_not?.exists?.field === 'package.deprecated' + ); + assert(deprecatedFilter, 'Should include deprecated filter'); + }); + + it('should not filter deprecated packages when searchExcludeDeprecated is false', async () => { + mock(app.config.cnpmcore, 'searchExcludeDeprecated', false); + let capturedQuery: any; + mockES.add( + { + method: 'POST', + path: `/${app.config.cnpmcore.elasticsearchIndex}/_search`, + }, + (params: any) => { + capturedQuery = params.body; + return { + hits: { + total: { value: 1, relation: 'eq' }, + hits: [ + { + _source: { + downloads: { all: 0 }, + package: { + name: 'deprecated-package', + deprecated: 'This package is deprecated', + }, + }, + }, + ], + }, + }; + } + ); + await app + .httpRequest() + .get('/-/v1/search?text=example&from=0&size=1') + .expect(200); + + // Verify that the query does not include deprecated filter + const filters = capturedQuery.query.function_score.query.bool.filter || []; + const deprecatedFilter = filters.find((f: any) => + f.bool?.must_not?.exists?.field === 'package.deprecated' + ); + assert(!deprecatedFilter, 'Should not include deprecated filter'); + }); + + it('should filter newly published packages when searchPackageMinAge is set', async () => { + mock(app.config.cnpmcore, 'searchPackageMinAge', '2w'); + let capturedQuery: any; + mockES.add( + { + method: 'POST', + path: `/${app.config.cnpmcore.elasticsearchIndex}/_search`, + }, + (params: any) => { + capturedQuery = params.body; + return { + hits: { + total: { value: 0, relation: 'eq' }, + hits: [], + }, + }; + } + ); + await app + .httpRequest() + .get('/-/v1/search?text=example&from=0&size=1') + .expect(200); + + // Verify that the query includes date range filter + assert(capturedQuery); + const filters = capturedQuery.query.function_score.query.bool.filter; + const dateFilter = filters.find((f: any) => f.range?.['package.date']); + assert(dateFilter, 'Should include date range filter'); + assert(dateFilter.range['package.date'].lte, 'Should have lte date threshold'); + }); + + it('should parse time string correctly in hours', async () => { + mock(app.config.cnpmcore, 'searchPackageMinAge', '48h'); + let capturedQuery: any; + mockES.add( + { + method: 'POST', + path: `/${app.config.cnpmcore.elasticsearchIndex}/_search`, + }, + (params: any) => { + capturedQuery = params.body; + return { + hits: { + total: { value: 0, relation: 'eq' }, + hits: [], + }, + }; + } + ); + await app + .httpRequest() + .get('/-/v1/search?text=example&from=0&size=1') + .expect(200); + + const filters = capturedQuery.query.function_score.query.bool.filter; + const dateFilter = filters.find((f: any) => f.range?.['package.date']); + assert(dateFilter, 'Should include date range filter for hours'); + }); + + it('should parse time string correctly in days', async () => { + mock(app.config.cnpmcore, 'searchPackageMinAge', '14d'); + let capturedQuery: any; + mockES.add( + { + method: 'POST', + path: `/${app.config.cnpmcore.elasticsearchIndex}/_search`, + }, + (params: any) => { + capturedQuery = params.body; + return { + hits: { + total: { value: 0, relation: 'eq' }, + hits: [], + }, + }; + } + ); + await app + .httpRequest() + .get('/-/v1/search?text=example&from=0&size=1') + .expect(200); + + const filters = capturedQuery.query.function_score.query.bool.filter; + const dateFilter = filters.find((f: any) => f.range?.['package.date']); + assert(dateFilter, 'Should include date range filter for days'); + }); }); describe('[PUT /-/v1/search/sync/:fullname] sync()', async () => {