From c6ef1efc0d5cc485a6deab1e76cee5780ebaa0aa Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Sat, 29 Jul 2023 18:57:02 +0100 Subject: [PATCH 01/15] #489: Added search endpoint using rudimentary search algorithm to calculate results based on a simple weighted score system. --- src/interfaces/SearchTermInterface.ts | 5 + .../global.character.aggregate.entity.ts | 14 ++ .../global/global.outfit.aggregate.entity.ts | 14 ++ .../controllers/rest.search.controller.ts | 153 ++++++++++++++++++ src/modules/rest/rest.module.ts | 2 + .../mongo/mongo.operations.service.ts | 18 +++ 6 files changed, 206 insertions(+) create mode 100644 src/interfaces/SearchTermInterface.ts create mode 100644 src/modules/rest/controllers/rest.search.controller.ts diff --git a/src/interfaces/SearchTermInterface.ts b/src/interfaces/SearchTermInterface.ts new file mode 100644 index 00000000..dfdd745a --- /dev/null +++ b/src/interfaces/SearchTermInterface.ts @@ -0,0 +1,5 @@ +export interface SearchTermInterface { + field: string; + term: string; + options: string; +} diff --git a/src/modules/data/entities/aggregate/global/global.character.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.character.aggregate.entity.ts index 764a414b..2833dae6 100644 --- a/src/modules/data/entities/aggregate/global/global.character.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.character.aggregate.entity.ts @@ -100,4 +100,18 @@ export default class GlobalCharacterAggregateEntity { default: Ps2AlertsEventType.LIVE_METAGAME, }) ps2AlertsEventType: Ps2AlertsEventType; + + @Exclude() + @ApiProperty({ + example: 100, + description: 'Search score weighting', + }) + searchScore?: number; + + @Exclude() + @ApiProperty({ + example: 'character', + description: 'Search result type', + }) + searchResultType?: string; } diff --git a/src/modules/data/entities/aggregate/global/global.outfit.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.outfit.aggregate.entity.ts index a0d0699b..34cf409b 100644 --- a/src/modules/data/entities/aggregate/global/global.outfit.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.outfit.aggregate.entity.ts @@ -107,4 +107,18 @@ export default class GlobalOutfitAggregateEntity { default: Ps2AlertsEventType.LIVE_METAGAME, }) ps2AlertsEventType: Ps2AlertsEventType; + + @Exclude() + @ApiProperty({ + example: 100, + description: 'Search score weighting', + }) + searchScore?: number; + + @Exclude() + @ApiProperty({ + example: 'character', + description: 'Search result type', + }) + searchResultType?: string; } diff --git a/src/modules/rest/controllers/rest.search.controller.ts b/src/modules/rest/controllers/rest.search.controller.ts new file mode 100644 index 00000000..22ca917c --- /dev/null +++ b/src/modules/rest/controllers/rest.search.controller.ts @@ -0,0 +1,153 @@ +import {Controller, Get, Inject, Optional, Query} from '@nestjs/common'; +import MongoOperationsService from '../../../services/mongo/mongo.operations.service'; +import {ApiOperation, ApiResponse, ApiTags} from '@nestjs/swagger'; +import {ApiImplicitQueries} from 'nestjs-swagger-api-implicit-queries-decorator'; +import {PAGINATION_IMPLICIT_QUERIES} from './common/rest.pagination.queries'; +import GlobalCharacterAggregateEntity from '../../data/entities/aggregate/global/global.character.aggregate.entity'; +import GlobalOutfitAggregateEntity from '../../data/entities/aggregate/global/global.outfit.aggregate.entity'; +import Pagination from '../../../services/mongo/pagination'; +import {SearchTermInterface} from '../../../interfaces/SearchTermInterface'; + +@ApiTags('Search') +@Controller('search') +export default class RestSearchController { + constructor( + @Inject(MongoOperationsService) private readonly mongoOperationsService: MongoOperationsService, + ) {} + + @Get() + @ApiOperation({summary: 'Searches GlobalCharacterAggregateEntity and GlobalOutfitAggregateEntity for a term'}) + @ApiImplicitQueries([...PAGINATION_IMPLICIT_QUERIES, { + name: 'type', + required: false, + description: 'The type of the data to be searched, either "characters" or "outfits". If not specified, both types will be searched.', + type: String, + enum: ['characters', 'outfits'], + }]) + @ApiResponse({ + status: 200, + description: 'The list of GlobalCharacterAggregateEntity and GlobalOutfitAggregateEntity for a search term', + type: Object, + isArray: false, + }) + async search( + @Query('searchTerm') searchTerm: string, + @Query('type') @Optional() type?: string, + @Query('sortBy') sortBy?: string, + @Query('order') order?: string, + ): Promise<{results: Array}> { + let characterResults: GlobalCharacterAggregateEntity[] = []; + let outfitResults: GlobalOutfitAggregateEntity[] = []; + + const pagination = new Pagination({sortBy, order, page: 0, pageSize: 10}, false); + + if (type === 'characters' || type === undefined) { + const characterSearchTerm: SearchTermInterface = { + field: 'character.name', + term: searchTerm, + options: 'i', + }; + characterResults = await this.mongoOperationsService.searchText( + GlobalCharacterAggregateEntity, + characterSearchTerm, + {$and: [{bracket: 0}]}, + pagination, + ); + } + + if (type === 'outfits' || type === undefined) { + const outfitNameSearchTerm: SearchTermInterface = { + field: 'outfit.name', + term: searchTerm, + options: 'i', + }; + const outfitTagSearchTerm: SearchTermInterface = { + field: 'outfit.tag', + term: searchTerm, + options: 'i', + }; + + // First, search for outfits by tag + const outfitTagResults = await this.mongoOperationsService.searchText( + GlobalOutfitAggregateEntity, + outfitTagSearchTerm, + {$and: [{bracket: 0}]}, + pagination, + ); + + const outfitNameResults = await this.mongoOperationsService.searchText( + GlobalOutfitAggregateEntity, + outfitNameSearchTerm, + {$and: [{bracket: 0}]}, + pagination, + ); + + // Combine both arrays, ensuring that the tag results appear first + outfitResults = [...outfitTagResults, ...outfitNameResults]; + + // Deduplicate outfits based on name + const outfitsMap = new Map(outfitResults.map((outfit) => [outfit.outfit.name, outfit])); + outfitResults = Array.from(outfitsMap.values()); + } + + const searchTermLower = searchTerm.toLowerCase(); + + // For outfits + outfitResults.forEach((outfit) => { + let score = 0; + + // Higher weight for exact matches on tag and name + if (outfit.outfit.tag?.toLowerCase() === searchTermLower || outfit.outfit.name.toLowerCase() === searchTermLower) { + score += 100; + } else { + // Lower weight for partial matches + const searchTermRegex = new RegExp(searchTerm, 'i'); + + if (outfit.outfit.tag?.match(searchTermRegex)) { + score += 20; + } else if (outfit.outfit.name.match(searchTermRegex)) { + score += 10; + } + } + + outfit.searchScore = score; + outfit.searchResultType = 'outfit'; // added type field + }); + + // For characters + characterResults.forEach((character) => { + let score = 0; + + // Higher weight for exact matches + if (character.character.name.toLowerCase() === searchTermLower) { + score += 100; + } else { + // Lower weight for partial matches + const searchTermRegex = new RegExp(searchTerm, 'i'); + + if (character.character.name.match(searchTermRegex)) { + score += 5; + } + } + + character.searchScore = score; + character.searchResultType = 'character'; // added type field + }); + + // Sort by score + characterResults.sort((a, b) => this.searchScores(a.searchScore, b.searchScore)); + + // combine both arrays + const results = [...characterResults, ...outfitResults].sort((a, b) => this.searchScores(a.searchScore, b.searchScore)); + + return {results}; + } + + private searchScores(a: number | undefined, b: number | undefined): number { + if (a && b) { + return b - a; + } + + return 0; + } +} diff --git a/src/modules/rest/rest.module.ts b/src/modules/rest/rest.module.ts index 627943de..1589f630 100644 --- a/src/modules/rest/rest.module.ts +++ b/src/modules/rest/rest.module.ts @@ -59,6 +59,7 @@ import {ConfigService} from '@nestjs/config'; import * as redisStore from 'cache-manager-ioredis'; import {RedisCacheService} from '../../services/cache/redis.cache.service'; import {AuthModule} from '../../auth/auth.module'; +import RestSearchController from './controllers/rest.search.controller'; /** * Handles incoming requests to the API via HTTP, CRUD environment. @@ -136,6 +137,7 @@ import {AuthModule} from '../../auth/auth.module'; RestInstanceFacilityControlController, RestInstanceMetagameController, RestOutfitwarsController, + RestSearchController, ], providers: [ {provide: APP_INTERCEPTOR, useClass: ClassSerializerInterceptor}, diff --git a/src/services/mongo/mongo.operations.service.ts b/src/services/mongo/mongo.operations.service.ts index 0b35f5f3..2a032955 100644 --- a/src/services/mongo/mongo.operations.service.ts +++ b/src/services/mongo/mongo.operations.service.ts @@ -174,6 +174,24 @@ export default class MongoOperationsService { } } + public async searchText(entity: new () => T, searchTerm?: {field: string, term: string, options: string}, filter?: object, pagination?: Pagination): Promise { + // Create a base filter with bracket = 0 + const baseFilter: { [key: string]: any } = {bracket: 0}; + + // If a search term is provided, add a regex query for the specified field + if (searchTerm) { + baseFilter[searchTerm.field] = {$regex: `^${searchTerm.term}.*`, $options: searchTerm.options}; + } + + // If an additional filter is provided, add it to the base filter + if (filter) { + Object.assign(baseFilter, filter); + } + + // Use the find() method with the filter and pagination options + return await this.em.find(entity, MongoOperationsService.createFindOptions(baseFilter, pagination)); + } + // eslint-disable-next-line @typescript-eslint/ban-types private static createFindOptions(filter?: {[k: string]: any}, pagination?: Pagination): object { let findOptions: {[k: string]: any} = {}; From ceeb71fbfd50324ebebf8c952f2e8ffff7d035bb Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Sat, 29 Jul 2023 20:47:11 +0100 Subject: [PATCH 02/15] #317: Removed results key --- src/modules/rest/controllers/rest.search.controller.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/modules/rest/controllers/rest.search.controller.ts b/src/modules/rest/controllers/rest.search.controller.ts index 22ca917c..f4ff495f 100644 --- a/src/modules/rest/controllers/rest.search.controller.ts +++ b/src/modules/rest/controllers/rest.search.controller.ts @@ -35,7 +35,7 @@ export default class RestSearchController { @Query('type') @Optional() type?: string, @Query('sortBy') sortBy?: string, @Query('order') order?: string, - ): Promise<{results: Array}> { + ): Promise> { let characterResults: GlobalCharacterAggregateEntity[] = []; let outfitResults: GlobalOutfitAggregateEntity[] = []; @@ -137,10 +137,8 @@ export default class RestSearchController { // Sort by score characterResults.sort((a, b) => this.searchScores(a.searchScore, b.searchScore)); - // combine both arrays - const results = [...characterResults, ...outfitResults].sort((a, b) => this.searchScores(a.searchScore, b.searchScore)); - - return {results}; + // Combine both arrays and return combined array sorted by score + return [...characterResults, ...outfitResults].sort((a, b) => this.searchScores(a.searchScore, b.searchScore)); } private searchScores(a: number | undefined, b: number | undefined): number { From 3a241f8471886b3ca76e1926bbb6d4bbcddf5fee Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Sun, 30 Jul 2023 21:57:38 +0100 Subject: [PATCH 03/15] #317: Created full search functionality which handles creation of a search index in Redis and updates it periodically with new players and outfits --- package.json | 1 + src/modules/cron/CronModule.ts | 2 + src/modules/cron/search.cron.ts | 187 +++++++++++++++++ .../global.character.aggregate.entity.ts | 17 +- .../global/global.outfit.aggregate.entity.ts | 14 -- .../controllers/healthcheck.controller.ts | 1 + .../controllers/rest.search.controller.ts | 196 +++++++----------- src/services/cache/redis.cache.service.ts | 27 ++- .../mongo/mongo.operations.service.ts | 9 +- src/services/mongo/pagination.ts | 12 +- yarn.lock | 25 +++ 11 files changed, 328 insertions(+), 163 deletions(-) create mode 100644 src/modules/cron/search.cron.ts diff --git a/package.json b/package.json index ceaff5dd..ea29ac31 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "cache-manager-ioredis": "^2.1.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", + "ioredis": "^5.3.2", "mongodb": "^4.14.0", "nestjs-swagger-api-implicit-queries-decorator": "^1.0.0", "passport": "^0.6.0", diff --git a/src/modules/cron/CronModule.ts b/src/modules/cron/CronModule.ts index 21e40b66..e4fc87f5 100644 --- a/src/modules/cron/CronModule.ts +++ b/src/modules/cron/CronModule.ts @@ -15,6 +15,7 @@ import {ConfigService} from '@nestjs/config'; import * as redisStore from 'cache-manager-ioredis'; import {RedisCacheService} from '../../services/cache/redis.cache.service'; import {XpmCron} from './xpm.cron'; +import {SearchCron} from './search.cron'; // import {OutfitWarsRankingsCron} from './outfitwars.rankings.cron'; // import OutfitwarsRankingEntity from '../data/entities/instance/outfitwars.ranking.entity'; @@ -48,6 +49,7 @@ import {XpmCron} from './xpm.cron'; BracketCron, // OutfitWarsRankingsCron, XpmCron, + SearchCron, ], }) export class CronModule {} diff --git a/src/modules/cron/search.cron.ts b/src/modules/cron/search.cron.ts new file mode 100644 index 00000000..13cefa3d --- /dev/null +++ b/src/modules/cron/search.cron.ts @@ -0,0 +1,187 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import {Inject, Injectable, Logger} from '@nestjs/common'; +import {Cron, CronExpression} from '@nestjs/schedule'; +import MongoOperationsService from '../../services/mongo/mongo.operations.service'; +import {RedisCacheService} from '../../services/cache/redis.cache.service'; +import GlobalCharacterAggregateEntity from '../data/entities/aggregate/global/global.character.aggregate.entity'; +import {Bracket} from '../data/ps2alerts-constants/bracket'; +import Pagination from '../../services/mongo/pagination'; +import GlobalOutfitAggregateEntity from '../data/entities/aggregate/global/global.outfit.aggregate.entity'; +import {pcWorldArray, World} from '../data/ps2alerts-constants/world'; +import {Ps2AlertsEventType} from '../data/ps2alerts-constants/ps2AlertsEventType'; + +@Injectable() +export class SearchCron { + private readonly logger = new Logger(SearchCron.name); + private readonly pageSize = 10000; + private readonly listPrefix = 'search'; + private readonly filter = {searchIndexed: false, bracket: Bracket.TOTAL, ps2AlertsEventType: Ps2AlertsEventType.LIVE_METAGAME}; + + constructor( + @Inject(MongoOperationsService) private readonly mongoOperationsService: MongoOperationsService, + private readonly cacheService: RedisCacheService, + ) {} + + @Cron(CronExpression.EVERY_5_SECONDS) + async handleCron(): Promise { + this.logger.log('Running Search sync job'); + + const lock = await this.cacheService.get('locks:search'); + + if (lock) { + this.logger.log('Search sync job already running'); + return; + } + + await this.cacheService.set('locks:search', Date.now(), 60 * 60); // 1 hour lock + + await this.syncCharacters(); + await this.syncOutfits(); + + await this.cacheService.unset('locks:search'); + + // @See CronHealthIndicator + // This sets the fact that the cron has run, so if it hasn't been run it will be terminated. + const key = '/crons/search'; + await this.cacheService.set(key, Date.now(), 595); // Just under 5 minutes = deadline for this cron + this.logger.debug('Set search cron run time'); + } + + async syncCharacters(): Promise { + this.logger.log('==== Syncing Characters ===='); + let page = 0; + let endOfRecords = false; + + const numberOfRecords = await this.mongoOperationsService.countDocuments(GlobalCharacterAggregateEntity, this.filter); + + if (numberOfRecords === 0) { + this.logger.log('No records to process'); + return; + } + + this.logger.log(`Found ${numberOfRecords} records to add to search cache`); + + // Loop through all Character records until we have less than 1000 returned + while (!endOfRecords) { + this.logger.log(`Processing records ${page * this.pageSize} -> ${(page * this.pageSize) + (this.pageSize - 1)}...`); + // Get all records that are not indexed + const records: GlobalCharacterAggregateEntity[] = await this.mongoOperationsService.findMany( + GlobalCharacterAggregateEntity, + this.filter, + new Pagination({page, pageSize: this.pageSize}), // We are purposefully NOT sorting here as it causes a full table scan and it's super fucking slow + ); + + if (records.length < this.pageSize) { + endOfRecords = true; + this.logger.log('At the end of character records'); + } + + // Loop through all records and add them to the cache + for await (const record of records) { + const environment = this.getEnvironment(record.world); + + // Store the lowercase version of the name acting as "normalized" for searching purposes + await this.cacheService.addDataToSortedSet(`${this.listPrefix}:${environment}:character_index`, [record.character.name.toLowerCase()], 0); + + // Create a key which contains the lowercase name as the key and the char ID as the value, which will be used by the search API to pull the record out of the DB. + // JSON.stringify is needed here as for some reason the client library favours using int64, this forces it to be a string + await this.cacheService.setPermanent(`${this.listPrefix}:${environment}:character_ids:${record.character.name.toLowerCase()}`, JSON.stringify(record.character.id)); + + // Mark the character as search indexed in the database to prevent being processed again + await this.mongoOperationsService.upsert(GlobalCharacterAggregateEntity, [{$set: {searchIndexed: true}}], [{'character.id': record.character.id}]); + } + + this.logger.log(`Added ${records.length} records to character search cache`); + this.logger.log(`${page * this.pageSize + records.length}/${numberOfRecords} processed`); + page++; + } + } + + async syncOutfits(): Promise { + this.logger.log('==== Syncing Outfits ===='); + let page = 0; + let endOfRecords = false; + let corruptOutfits = 0; + + const numberOfRecords = await this.mongoOperationsService.countDocuments(GlobalOutfitAggregateEntity, this.filter); + + if (numberOfRecords === 0) { + this.logger.log('No records to process'); + return; + } + + this.logger.log(`Found ${numberOfRecords} records to add to search cache`); + + // Loop through all Character records until we have less than 1000 returned + while (!endOfRecords) { + this.logger.log(`Processing records ${page * this.pageSize} -> ${(page * this.pageSize) + (this.pageSize - 1)}...`); + + // Get all records that are not indexed + const records: GlobalOutfitAggregateEntity[] = await this.mongoOperationsService.findMany( + GlobalOutfitAggregateEntity, + this.filter, + new Pagination({page, pageSize: this.pageSize}), // We are purposefully NOT sorting here as it causes a full table scan and it's super fucking slow + ); + + if (records.length < this.pageSize) { + endOfRecords = true; + this.logger.log('At the end of outfit records'); + } + + // Loop through all records and add them to the cache + for await (const record of records) { + const environment = this.getEnvironment(record.world); + + // Handle outfit corruptions that come up occasionally + if (!record.outfit.name || !record.outfit.id) { + this.logger.error('Corrupt outfit detected!'); + corruptOutfits++; + + try { + await this.mongoOperationsService.deleteOne(GlobalOutfitAggregateEntity, {_id: record._id}); + } catch (err) { + this.logger.error(err); + } + + continue; + } + + await this.cacheService.addDataToSortedSet(`${this.listPrefix}:${environment}:outfit_index`, [record.outfit.name.toLowerCase()]); + + if (record.outfit.tag) { + await this.cacheService.addDataToSortedSet(`${this.listPrefix}:${environment}:outfit_tag_index`, [record.outfit.tag.toLowerCase()]); + } + + // Create a key which contains the lowercase name as the key and the outfit ID as the value, which will be used by the search API to pull the record out of the DB + // JSON.stringify is needed here as for some reason the client library favours using int64, this forces it to be a string + await this.cacheService.setPermanent(`${this.listPrefix}:${environment}:outfit_ids:${record.outfit.name.toLowerCase()}`, JSON.stringify(record.outfit.id)); + + // Do the same for tag if it exists + if (record.outfit.tag) { + await this.cacheService.setPermanent(`${this.listPrefix}:${environment}:outfit_ids_tag:${record.outfit.tag.toLowerCase()}`, JSON.stringify(record.outfit.id)); + } + + // Mark the character as search indexed in the database + await this.mongoOperationsService.upsert(GlobalOutfitAggregateEntity, [{$set: {searchIndexed: true}}], [{'outfit.id': record.outfit.id}]); + } + + this.logger.log(`Added ${records.length} records to outfit search cache`); + this.logger.log(`${page * this.pageSize + records.length}/${numberOfRecords} processed`); + this.logger.error(`Corrupt outfits: ${corruptOutfits}`); + + page++; + } + } + + getEnvironment(world: World): string { + if (pcWorldArray.includes(world)) { + return 'pc'; + } else if (world === World.CERES) { + return 'ps4_eu'; + } else { + return 'ps4_us'; + } + + return 'UNKNOWN'; + } +} diff --git a/src/modules/data/entities/aggregate/global/global.character.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.character.aggregate.entity.ts index 2833dae6..77cc0039 100644 --- a/src/modules/data/entities/aggregate/global/global.character.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.character.aggregate.entity.ts @@ -101,17 +101,10 @@ export default class GlobalCharacterAggregateEntity { }) ps2AlertsEventType: Ps2AlertsEventType; - @Exclude() - @ApiProperty({ - example: 100, - description: 'Search score weighting', - }) - searchScore?: number; - - @Exclude() - @ApiProperty({ - example: 'character', - description: 'Search result type', + @ApiProperty({example: true, description: 'Denotes if this aggregate is indexed for searching'}) + @Column({ + type: 'boolean', + default: false, }) - searchResultType?: string; + searchIndexed: boolean; } diff --git a/src/modules/data/entities/aggregate/global/global.outfit.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.outfit.aggregate.entity.ts index 34cf409b..a0d0699b 100644 --- a/src/modules/data/entities/aggregate/global/global.outfit.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.outfit.aggregate.entity.ts @@ -107,18 +107,4 @@ export default class GlobalOutfitAggregateEntity { default: Ps2AlertsEventType.LIVE_METAGAME, }) ps2AlertsEventType: Ps2AlertsEventType; - - @Exclude() - @ApiProperty({ - example: 100, - description: 'Search score weighting', - }) - searchScore?: number; - - @Exclude() - @ApiProperty({ - example: 'character', - description: 'Search result type', - }) - searchResultType?: string; } diff --git a/src/modules/healthcheck/controllers/healthcheck.controller.ts b/src/modules/healthcheck/controllers/healthcheck.controller.ts index 5b4e4e0c..cc00e7ba 100644 --- a/src/modules/healthcheck/controllers/healthcheck.controller.ts +++ b/src/modules/healthcheck/controllers/healthcheck.controller.ts @@ -66,6 +66,7 @@ export default class HealthcheckController { indicators.push(async () => this.cronHealth.isHealthy('combatHistory', 65)); // indicators.push(async () => this.cronHealth.isHealthy('outfitwarsrankings', 60 * 60 * 24 + 300)); indicators.push(async () => this.cronHealth.isHealthy('xpm', 35)); + indicators.push(async () => this.cronHealth.isHealthy('search', 605)); } return this.health.check(indicators); diff --git a/src/modules/rest/controllers/rest.search.controller.ts b/src/modules/rest/controllers/rest.search.controller.ts index f4ff495f..1e42c1df 100644 --- a/src/modules/rest/controllers/rest.search.controller.ts +++ b/src/modules/rest/controllers/rest.search.controller.ts @@ -1,151 +1,99 @@ -import {Controller, Get, Inject, Optional, Query} from '@nestjs/common'; +import {Controller, Get, Inject, Query} from '@nestjs/common'; import MongoOperationsService from '../../../services/mongo/mongo.operations.service'; import {ApiOperation, ApiResponse, ApiTags} from '@nestjs/swagger'; -import {ApiImplicitQueries} from 'nestjs-swagger-api-implicit-queries-decorator'; -import {PAGINATION_IMPLICIT_QUERIES} from './common/rest.pagination.queries'; import GlobalCharacterAggregateEntity from '../../data/entities/aggregate/global/global.character.aggregate.entity'; import GlobalOutfitAggregateEntity from '../../data/entities/aggregate/global/global.outfit.aggregate.entity'; -import Pagination from '../../../services/mongo/pagination'; -import {SearchTermInterface} from '../../../interfaces/SearchTermInterface'; +import {Ps2AlertsEventType} from '../../data/ps2alerts-constants/ps2AlertsEventType'; +import {RedisCacheService} from '../../../services/cache/redis.cache.service'; +import {Bracket} from '../../data/ps2alerts-constants/bracket'; @ApiTags('Search') @Controller('search') export default class RestSearchController { + private readonly environments = ['pc', 'ps4_eu', 'ps4_us']; constructor( @Inject(MongoOperationsService) private readonly mongoOperationsService: MongoOperationsService, + private readonly cacheService: RedisCacheService, ) {} - @Get() - @ApiOperation({summary: 'Searches GlobalCharacterAggregateEntity and GlobalOutfitAggregateEntity for a term'}) - @ApiImplicitQueries([...PAGINATION_IMPLICIT_QUERIES, { - name: 'type', - required: false, - description: 'The type of the data to be searched, either "characters" or "outfits". If not specified, both types will be searched.', - type: String, - enum: ['characters', 'outfits'], - }]) + @Get('characters') + @ApiOperation({summary: 'Searches GlobalCharacterAggregateEntity for a term'}) @ApiResponse({ status: 200, - description: 'The list of GlobalCharacterAggregateEntity and GlobalOutfitAggregateEntity for a search term', + description: 'The list of GlobalCharacterAggregateEntity for a search term', type: Object, - isArray: false, + isArray: true, }) - async search( + async searchCharacters( @Query('searchTerm') searchTerm: string, - @Query('type') @Optional() type?: string, - @Query('sortBy') sortBy?: string, - @Query('order') order?: string, - ): Promise> { - let characterResults: GlobalCharacterAggregateEntity[] = []; - let outfitResults: GlobalOutfitAggregateEntity[] = []; - - const pagination = new Pagination({sortBy, order, page: 0, pageSize: 10}, false); - - if (type === 'characters' || type === undefined) { - const characterSearchTerm: SearchTermInterface = { - field: 'character.name', - term: searchTerm, - options: 'i', - }; - characterResults = await this.mongoOperationsService.searchText( - GlobalCharacterAggregateEntity, - characterSearchTerm, - {$and: [{bracket: 0}]}, - pagination, - ); - } - - if (type === 'outfits' || type === undefined) { - const outfitNameSearchTerm: SearchTermInterface = { - field: 'outfit.name', - term: searchTerm, - options: 'i', - }; - const outfitTagSearchTerm: SearchTermInterface = { - field: 'outfit.tag', - term: searchTerm, - options: 'i', - }; - - // First, search for outfits by tag - const outfitTagResults = await this.mongoOperationsService.searchText( - GlobalOutfitAggregateEntity, - outfitTagSearchTerm, - {$and: [{bracket: 0}]}, - pagination, - ); - - const outfitNameResults = await this.mongoOperationsService.searchText( - GlobalOutfitAggregateEntity, - outfitNameSearchTerm, - {$and: [{bracket: 0}]}, - pagination, - ); - - // Combine both arrays, ensuring that the tag results appear first - outfitResults = [...outfitTagResults, ...outfitNameResults]; - - // Deduplicate outfits based on name - const outfitsMap = new Map(outfitResults.map((outfit) => [outfit.outfit.name, outfit])); - outfitResults = Array.from(outfitsMap.values()); - } - - const searchTermLower = searchTerm.toLowerCase(); - - // For outfits - outfitResults.forEach((outfit) => { - let score = 0; - - // Higher weight for exact matches on tag and name - if (outfit.outfit.tag?.toLowerCase() === searchTermLower || outfit.outfit.name.toLowerCase() === searchTermLower) { - score += 100; - } else { - // Lower weight for partial matches - const searchTermRegex = new RegExp(searchTerm, 'i'); - - if (outfit.outfit.tag?.match(searchTermRegex)) { - score += 20; - } else if (outfit.outfit.name.match(searchTermRegex)) { - score += 10; - } + ): Promise { + // Time for some voodoo + const characterIds: string[] = []; + + // Loop through each environment and perform a prefix search via Redis using an insensitive version of the search term. This will return the names of the characters that match the search term. + for (const environment of this.environments) { + const nameListKey = `search:${environment}:character_index`; + const characterNames = await this.cacheService.searchDataInSortedSet(nameListKey, searchTerm.toLowerCase()); + + // Now we have the character names, we need to grab their IDs by performing a lookup by lowercase name in the database + for (const characterName of characterNames) { + characterIds.push(String(await this.cacheService.get(`search:${environment}:character_ids:${characterName}`))); } + } - outfit.searchScore = score; - outfit.searchResultType = 'outfit'; // added type field + // Now we have a list of character IDs to grab, we now need to actually grab the characters from the database + return await this.mongoOperationsService.findMany(GlobalCharacterAggregateEntity, { + 'character.id': {$in: characterIds}, + bracket: Bracket.TOTAL, + ps2AlertsEventType: Ps2AlertsEventType.LIVE_METAGAME, }); + } - // For characters - characterResults.forEach((character) => { - let score = 0; - - // Higher weight for exact matches - if (character.character.name.toLowerCase() === searchTermLower) { - score += 100; - } else { - // Lower weight for partial matches - const searchTermRegex = new RegExp(searchTerm, 'i'); - - if (character.character.name.match(searchTermRegex)) { - score += 5; - } + @Get('outfits') + @ApiOperation({summary: 'Searches GlobalOutfitAggregateEntity for a term'}) + @ApiResponse({ + status: 200, + description: 'The list of GlobalOutfitAggregateEntity for a search term', + type: Object, + isArray: false, + }) + async searchOutfits( + @Query('searchTerm') searchTerm: string, + ): Promise { + // Time for some extra voodoo + let outfitIds: string[] = []; + + // Loop through each environment and perform a prefix search via Redis using an insensitive version of the search term. This will return the names of the outfits that match the search term. + for (const environment of this.environments) { + const nameListKey = `search:${environment}:outfit_index`; + const tagListKey = `search:${environment}:outfit_tag_index`; + const outfitNames = await this.cacheService.searchDataInSortedSet(nameListKey, searchTerm.toLowerCase()); + const outfitTags = await this.cacheService.searchDataInSortedSet(tagListKey, searchTerm.toLowerCase()); + + // Now we have the character names, we need to grab their IDs by performing a lookup by lowercase name in the database + for (const outfitName of outfitNames) { + outfitIds.push( + String(await this.cacheService.get(`search:${environment}:outfit_ids:${outfitName}`)), + ); } - character.searchScore = score; - character.searchResultType = 'character'; // added type field - }); - - // Sort by score - characterResults.sort((a, b) => this.searchScores(a.searchScore, b.searchScore)); - - // Combine both arrays and return combined array sorted by score - return [...characterResults, ...outfitResults].sort((a, b) => this.searchScores(a.searchScore, b.searchScore)); - } - - private searchScores(a: number | undefined, b: number | undefined): number { - if (a && b) { - return b - a; + // We also need to search on outfit tag for possible hits + for (const outfitTag of outfitTags) { + outfitIds.push( + String(await this.cacheService.get(`search:${environment}:outfit_ids_tag:${outfitTag}`)), + ); + } } - return 0; + // Deduplicate the outfit IDs + const outfitIdsSet = new Set(outfitIds); + outfitIds = Array.from(outfitIdsSet); + + // Now we have a list of character IDs to grab, we now need to actually grab the characters from the database + return await this.mongoOperationsService.findMany(GlobalOutfitAggregateEntity, { + 'outfit.id': {$in: outfitIds}, + bracket: Bracket.TOTAL, + ps2AlertsEventType: Ps2AlertsEventType.LIVE_METAGAME, + }); } } diff --git a/src/services/cache/redis.cache.service.ts b/src/services/cache/redis.cache.service.ts index e0185ae6..48ff3478 100644 --- a/src/services/cache/redis.cache.service.ts +++ b/src/services/cache/redis.cache.service.ts @@ -1,20 +1,45 @@ import {CACHE_MANAGER, Inject, Injectable} from '@nestjs/common'; import {Cache} from 'cache-manager'; +import * as Redis from 'ioredis'; @Injectable() export class RedisCacheService { + private readonly redisClient: Redis.Redis; constructor( @Inject(CACHE_MANAGER) private readonly cache: Cache, - ) {} + ) { + // Yay for packages that don't have full support for TS >:( + // eslint-disable-next-line @typescript-eslint/no-unsafe-call,@typescript-eslint/no-explicit-any,@typescript-eslint/no-unsafe-member-access + this.redisClient = (this.cache.store as any).getClient() as Redis.Redis; + } async set(key: string, data: T, ttl = 3600): Promise { await this.cache.set(key, data, {ttl}); return data; } + async setPermanent(key: string, data: string): Promise { + await this.redisClient.set(key, data); + return data; + } + + async unset(key: string): Promise { + await this.cache.del(key); + } + async get(key: string): Promise { const data: T | null = await this.cache.get(key) ?? null; return data ?? null; } + + async addDataToSortedSet(key: string, data: string[], score = 0): Promise { + // Flattening array of [score, data] pairs + const args: Array = data.reduce>((arr, item) => [...arr, score, item], []); + await this.redisClient.zadd(key, ...args); + } + + async searchDataInSortedSet(key: string, prefix: string): Promise { + return this.redisClient.zrangebylex(key, `[${prefix}`, `[${prefix}\xff`); + } } diff --git a/src/services/mongo/mongo.operations.service.ts b/src/services/mongo/mongo.operations.service.ts index 2a032955..52d24e94 100644 --- a/src/services/mongo/mongo.operations.service.ts +++ b/src/services/mongo/mongo.operations.service.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types,@typescript-eslint/no-explicit-any,@typescript-eslint/no-unsafe-return,@typescript-eslint/no-unsafe-assignment,@typescript-eslint/no-unsafe-argument */ -import {CollectionAggregationOptions, MongoEntityManager, ObjectID, ObjectLiteral} from 'typeorm'; +import {CollectionAggregationOptions, FindOptionsWhere, MongoEntityManager, ObjectID, ObjectLiteral} from 'typeorm'; import {InjectEntityManager} from '@nestjs/typeorm'; import {Injectable} from '@nestjs/common'; import Pagination from './pagination'; @@ -176,7 +176,7 @@ export default class MongoOperationsService { public async searchText(entity: new () => T, searchTerm?: {field: string, term: string, options: string}, filter?: object, pagination?: Pagination): Promise { // Create a base filter with bracket = 0 - const baseFilter: { [key: string]: any } = {bracket: 0}; + const baseFilter: { [key: string]: any } = {}; // If a search term is provided, add a regex query for the specified field if (searchTerm) { @@ -192,6 +192,11 @@ export default class MongoOperationsService { return await this.em.find(entity, MongoOperationsService.createFindOptions(baseFilter, pagination)); } + public async countDocuments(entity: new () => T, filter: FindOptionsWhere): Promise { + const repository = this.em.getRepository(entity); + return repository.countBy(filter); + } + // eslint-disable-next-line @typescript-eslint/ban-types private static createFindOptions(filter?: {[k: string]: any}, pagination?: Pagination): object { let findOptions: {[k: string]: any} = {}; diff --git a/src/services/mongo/pagination.ts b/src/services/mongo/pagination.ts index f05d09c3..584da966 100644 --- a/src/services/mongo/pagination.ts +++ b/src/services/mongo/pagination.ts @@ -6,16 +6,8 @@ export default class Pagination { public constructor(pageQuery: {sortBy?: string, order?: string, pageSize?: number, page?: number}, limited = false) { this.take = 100; - if (pageQuery.pageSize) { - if (pageQuery.pageSize < 1000) { - this.take = pageQuery.pageSize; - } else { - this.take = 1000; - } - } - - if (!limited && !pageQuery.pageSize) { - this.take = undefined; + if (!limited && pageQuery.pageSize) { + this.take = pageQuery.pageSize; } if (pageQuery.pageSize && pageQuery.page) { diff --git a/yarn.lock b/yarn.lock index 461e74ab..0fdb8fe0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -981,6 +981,11 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== +"@ioredis/commands@^1.1.1": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" + integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg== + "@jridgewell/gen-mapping@^0.3.0": version "0.3.2" resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz#c1aedc61e853f2bb9f5dfe6d4442d3b565b253b9" @@ -2596,6 +2601,11 @@ denque@^1.1.0: resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf" integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw== +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" @@ -3787,6 +3797,21 @@ ioredis@^4.14.1: redis-parser "^3.0.0" standard-as-callback "^2.1.0" +ioredis@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7" + integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA== + dependencies: + "@ioredis/commands" "^1.1.1" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + ip@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.0.tgz#4cf4ab182fee2314c75ede1276f8c80b479936da" From c0229447f417a2a8086c5b665b036badd5816ff5 Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Sun, 30 Jul 2023 23:22:20 +0100 Subject: [PATCH 04/15] #317: Updated cron to minutely --- src/modules/cron/search.cron.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/cron/search.cron.ts b/src/modules/cron/search.cron.ts index 13cefa3d..a0814325 100644 --- a/src/modules/cron/search.cron.ts +++ b/src/modules/cron/search.cron.ts @@ -22,7 +22,7 @@ export class SearchCron { private readonly cacheService: RedisCacheService, ) {} - @Cron(CronExpression.EVERY_5_SECONDS) + @Cron(CronExpression.EVERY_MINUTE) async handleCron(): Promise { this.logger.log('Running Search sync job'); @@ -59,7 +59,7 @@ export class SearchCron { return; } - this.logger.log(`Found ${numberOfRecords} records to add to search cache`); + this.logger.log(`Found ${numberOfRecords} records to add to search character cache`); // Loop through all Character records until we have less than 1000 returned while (!endOfRecords) { From e80e6547d6f131a840b566338bcea1cc36051428 Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Mon, 31 Jul 2023 13:55:01 +0100 Subject: [PATCH 05/15] Updated caching, fixed pagination logic --- .../global/rest.aggregate.global.character.controller.ts | 4 ++-- .../global/rest.aggregate.global.victory.controller.ts | 5 ++--- .../rest/controllers/rest.instance.metagame.controller.ts | 8 ++++++-- src/services/mongo/pagination.ts | 8 ++++++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.character.controller.ts b/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.character.controller.ts index 54e43c15..d250bd4d 100644 --- a/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.character.controller.ts +++ b/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.character.controller.ts @@ -56,7 +56,7 @@ export default class RestGlobalCharacterAggregateController extends BaseGlobalAg return await this.cacheService.get(key) ?? await this.cacheService.set( key, await this.mongoOperationsService.findMany(GlobalCharacterAggregateEntity, {world, bracket, ps2AlertsEventType}, pagination), - 900); + 60 * 15); } @Get('global/character/:character') @@ -75,7 +75,7 @@ export default class RestGlobalCharacterAggregateController extends BaseGlobalAg bracket = this.correctBracket(bracket, ps2AlertsEventType); // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - const key = `/global/character/${character}/B:${bracket}-ET:${ps2AlertsEventType}`; + const key = `cache:endpoints:character:${character}-B:${bracket}-ET:${ps2AlertsEventType}`; // eslint-disable-next-line @typescript-eslint/no-unsafe-return return await this.cacheService.get(key) ?? await this.cacheService.set( diff --git a/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.victory.controller.ts b/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.victory.controller.ts index db2ea69d..76812f53 100644 --- a/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.victory.controller.ts +++ b/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.victory.controller.ts @@ -64,15 +64,14 @@ export default class RestGlobalVictoryAggregateController extends BaseGlobalAggr date: new Range('date', dateFrom, dateTo).build(), }; - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - const key = `/global/victories/W:${world}-Z:${zone}-B:${bracket}-ET:${ps2AlertsEventType}?DF:${dateFrom}-DT:${dateTo}`; + const key = `cache:endpoints:victories:W:${world ?? 0}-Z:${zone ?? 0}-B:${bracket ?? 0}-ET:${ps2AlertsEventType ?? 0}-DF:${dateFrom ? dateFrom.toString() : 0}-DT:${dateTo ? dateTo.toString() : 0}`; const pagination = new Pagination({sortBy: 'date', order: 'desc'}); // eslint-disable-next-line @typescript-eslint/no-unsafe-return return await this.cacheService.get(key) ?? await this.cacheService.set( key, await this.mongoOperationsService.findMany(GlobalVictoryAggregate, filter, pagination), - 60, + 60 * 15, ); } } diff --git a/src/modules/rest/controllers/rest.instance.metagame.controller.ts b/src/modules/rest/controllers/rest.instance.metagame.controller.ts index b92de35e..40b68fb2 100644 --- a/src/modules/rest/controllers/rest.instance.metagame.controller.ts +++ b/src/modules/rest/controllers/rest.instance.metagame.controller.ts @@ -184,7 +184,11 @@ export class RestInstanceMetagameController { 'result.victor': victor ?? undefined, }; - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return await this.mongoOperationsService.findMany(InstanceMetagameTerritoryEntity, filter, new Pagination({sortBy, order, page, pageSize}, false)); + const key = `cache:endpoints:instance-metagame:W-65${world ?? 0}-Z:${zone ?? 0}-TSF:${timeStartedFrom ? timeStartedFrom.toString() : 0}-TST:${timeStartedTo ? timeStartedTo.toString() : 0}-B:${bracket ?? 0}-V:${victor ?? 0}-P:${page ?? 0}-PS:${pageSize ?? 0}`; + + return await this.cacheService.get(key) ?? await this.cacheService.set( + key, + await this.mongoOperationsService.findMany(InstanceMetagameTerritoryEntity, filter, new Pagination({sortBy, order, page, pageSize}, false)), + 60 * 15); } } diff --git a/src/services/mongo/pagination.ts b/src/services/mongo/pagination.ts index 584da966..17958cf7 100644 --- a/src/services/mongo/pagination.ts +++ b/src/services/mongo/pagination.ts @@ -6,8 +6,12 @@ export default class Pagination { public constructor(pageQuery: {sortBy?: string, order?: string, pageSize?: number, page?: number}, limited = false) { this.take = 100; - if (!limited && pageQuery.pageSize) { - this.take = pageQuery.pageSize; + if (!limited) { + if (pageQuery.pageSize) { + this.take = pageQuery.pageSize; + } else { + this.take = undefined; + } } if (pageQuery.pageSize && pageQuery.page) { From aef359f4c42428f0cb1a1c3d412cd9403a6ed87d Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Mon, 31 Jul 2023 14:12:49 +0100 Subject: [PATCH 06/15] Updated constants --- src/modules/data/ps2alerts-constants | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/data/ps2alerts-constants b/src/modules/data/ps2alerts-constants index 4e6bcfe1..661c340d 160000 --- a/src/modules/data/ps2alerts-constants +++ b/src/modules/data/ps2alerts-constants @@ -1 +1 @@ -Subproject commit 4e6bcfe115c723106d41b3b270afb70c4c0adde6 +Subproject commit 661c340df51e4c4f9965bd7e1d96ab14001d7d7b From c55cc3f5e470de80628084fe35a5cec8fc079d54 Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Mon, 31 Jul 2023 14:26:04 +0100 Subject: [PATCH 07/15] Fixed invalid relative pathing --- .../aggregate/global/global.character.aggregate.entity.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/data/entities/aggregate/global/global.character.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.character.aggregate.entity.ts index 77cc0039..43f7d6f2 100644 --- a/src/modules/data/entities/aggregate/global/global.character.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.character.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import {World, worldArray} from '../../../ps2alerts-constants/world'; @@ -25,6 +25,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class GlobalCharacterAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({enum: worldArray, example: 10, description: 'Server / World ID'}) From 5837aba862ababb28881c54ad511a31964d5b20b Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Mon, 31 Jul 2023 14:26:38 +0100 Subject: [PATCH 08/15] Updated constants --- src/modules/data/ps2alerts-constants | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/data/ps2alerts-constants b/src/modules/data/ps2alerts-constants index 661c340d..dacca118 160000 --- a/src/modules/data/ps2alerts-constants +++ b/src/modules/data/ps2alerts-constants @@ -1 +1 @@ -Subproject commit 661c340df51e4c4f9965bd7e1d96ab14001d7d7b +Subproject commit dacca1186f5df32d8696f94678f6c70d71d711b9 From 73c5e6b20f84f8b8448177079587a944ebb04672 Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Mon, 31 Jul 2023 14:28:43 +0100 Subject: [PATCH 09/15] Hide all mongo _ids from the API --- .../global/global.facility.control.aggregate.entity.ts | 3 ++- .../aggregate/global/global.faction.combat.aggregate.entity.ts | 3 ++- .../aggregate/global/global.loadout.aggregate.entity.ts | 3 ++- .../aggregate/global/global.outfit.aggregate.entity.ts | 3 ++- .../aggregate/global/global.vehicle.aggregate.entity.ts | 3 ++- .../global/global.vehicle.character.aggregate.entity.ts | 3 ++- .../aggregate/global/global.victory.aggregate.entity.ts | 3 ++- .../aggregate/global/global.weapon.aggregate.entity.ts | 3 ++- .../aggregate/instance/instance.character.aggregate.entity.ts | 3 ++- .../instance/instance.combat.history.aggregate.entity.ts | 3 ++- .../instance/instance.facility.control.aggregate.entity.ts | 3 ++- .../instance/instance.faction.combat.aggregate.entity.ts | 3 ++- .../aggregate/instance/instance.loadout.aggregate.entity.ts | 3 ++- .../aggregate/instance/instance.outfit.aggregate.entity.ts | 3 ++- .../aggregate/instance/instance.population.aggregate.entity.ts | 3 ++- .../instance/instance.population.averages.aggregate.entity.ts | 3 ++- .../aggregate/instance/instance.vehicle.aggregate.entity.ts | 3 ++- .../instance/instance.vehicle.character.aggregate.entity.ts | 3 ++- .../aggregate/instance/instance.weapon.aggregate.entity.ts | 3 ++- 19 files changed, 38 insertions(+), 19 deletions(-) diff --git a/src/modules/data/entities/aggregate/global/global.facility.control.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.facility.control.aggregate.entity.ts index a14c6d91..664c14ee 100644 --- a/src/modules/data/entities/aggregate/global/global.facility.control.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.facility.control.aggregate.entity.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {World, worldArray} from '../../../ps2alerts-constants/world'; import FacilityFactionControl from '../common/facility.faction.control.embed'; @@ -18,6 +18,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class GlobalFacilityControlAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({enum: worldArray, example: 10, description: 'Server / World ID'}) diff --git a/src/modules/data/entities/aggregate/global/global.faction.combat.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.faction.combat.aggregate.entity.ts index 9e949bcc..6bdd33fc 100644 --- a/src/modules/data/entities/aggregate/global/global.faction.combat.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.faction.combat.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, ObjectID, Index} from 'typeorm'; import {World, worldArray} from '../../../ps2alerts-constants/world'; @@ -18,6 +18,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class GlobalFactionCombatAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({enum: worldArray, example: 10, description: 'Server / World ID'}) diff --git a/src/modules/data/entities/aggregate/global/global.loadout.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.loadout.aggregate.entity.ts index fb6b6310..f135d799 100644 --- a/src/modules/data/entities/aggregate/global/global.loadout.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.loadout.aggregate.entity.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Loadout, loadoutArray} from '../../../ps2alerts-constants/loadout'; import {World, worldArray} from '../../../ps2alerts-constants/world'; @@ -18,6 +18,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class GlobalLoadoutAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({enum: worldArray, example: 10, description: 'Server / World ID'}) diff --git a/src/modules/data/entities/aggregate/global/global.outfit.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.outfit.aggregate.entity.ts index a0d0699b..86e9d599 100644 --- a/src/modules/data/entities/aggregate/global/global.outfit.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.outfit.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import {World, worldArray} from '../../../ps2alerts-constants/world'; @@ -25,6 +25,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class GlobalOutfitAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({type: OutfitEmbed, description: 'Outfit details'}) diff --git a/src/modules/data/entities/aggregate/global/global.vehicle.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.vehicle.aggregate.entity.ts index fffa7c4f..8da3f606 100644 --- a/src/modules/data/entities/aggregate/global/global.vehicle.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.vehicle.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import VehicleStatsEmbed from '../common/vehicle.vs.vehicle.embed'; @@ -17,6 +17,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class GlobalVehicleAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({example: Vehicle.FLASH, enum: vehicleArray, description: 'Vehicle ID'}) diff --git a/src/modules/data/entities/aggregate/global/global.vehicle.character.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.vehicle.character.aggregate.entity.ts index 9f4c3635..9929c13f 100644 --- a/src/modules/data/entities/aggregate/global/global.vehicle.character.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.vehicle.character.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import VehicleStatsEmbed from '../common/vehicle.vs.vehicle.embed'; @@ -18,6 +18,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class GlobalVehicleCharacterAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({example: World.MILLER, enum: worldArray, description: 'World ID'}) diff --git a/src/modules/data/entities/aggregate/global/global.victory.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.victory.aggregate.entity.ts index 54eece68..b27da644 100644 --- a/src/modules/data/entities/aggregate/global/global.victory.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.victory.aggregate.entity.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {World, worldArray} from '../../../ps2alerts-constants/world'; import {Zone, zoneArray} from '../../../ps2alerts-constants/zone'; @@ -17,6 +17,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class GlobalVictoryAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({enum: worldArray, example: 10, description: 'Server / World ID'}) diff --git a/src/modules/data/entities/aggregate/global/global.weapon.aggregate.entity.ts b/src/modules/data/entities/aggregate/global/global.weapon.aggregate.entity.ts index 8eeebeed..6ae5f4f2 100644 --- a/src/modules/data/entities/aggregate/global/global.weapon.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/global/global.weapon.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import {World, worldArray} from '../../../ps2alerts-constants/world'; @@ -22,6 +22,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class GlobalWeaponAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({enum: worldArray, example: 10, description: 'Server / World ID'}) diff --git a/src/modules/data/entities/aggregate/instance/instance.character.aggregate.entity.ts b/src/modules/data/entities/aggregate/instance/instance.character.aggregate.entity.ts index a24ef71f..967d89c8 100644 --- a/src/modules/data/entities/aggregate/instance/instance.character.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/instance/instance.character.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import CharacterEmbed from '../common/character.embed'; @@ -16,6 +16,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class InstanceCharacterAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({example: '10-12345', description: 'The Server-CensusInstanceId combination'}) diff --git a/src/modules/data/entities/aggregate/instance/instance.combat.history.aggregate.entity.ts b/src/modules/data/entities/aggregate/instance/instance.combat.history.aggregate.entity.ts index f6c4048b..90dd6b7e 100644 --- a/src/modules/data/entities/aggregate/instance/instance.combat.history.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/instance/instance.combat.history.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import CombatStats from '../common/combat.stats.embed'; @@ -12,6 +12,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class InstanceCombatHistoryAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({example: '10-12345', description: 'The Server-CensusInstanceId combination'}) diff --git a/src/modules/data/entities/aggregate/instance/instance.facility.control.aggregate.entity.ts b/src/modules/data/entities/aggregate/instance/instance.facility.control.aggregate.entity.ts index 59677d08..3ac9cfa9 100644 --- a/src/modules/data/entities/aggregate/instance/instance.facility.control.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/instance/instance.facility.control.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import FacilityFactionControl from '../common/facility.faction.control.embed'; @@ -15,6 +15,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class InstanceFacilityControlAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({example: '10-12345', description: 'The Server-CensusInstanceId combination'}) diff --git a/src/modules/data/entities/aggregate/instance/instance.faction.combat.aggregate.entity.ts b/src/modules/data/entities/aggregate/instance/instance.faction.combat.aggregate.entity.ts index 1b05307c..c92afc7f 100644 --- a/src/modules/data/entities/aggregate/instance/instance.faction.combat.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/instance/instance.faction.combat.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, ObjectID, Index} from 'typeorm'; import CombatStats from '../common/combat.stats.embed'; @@ -14,6 +14,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class InstanceFactionCombatAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id?: ObjectID; @ApiProperty({example: '10-12345', description: 'The Server-CensusInstanceId combination'}) diff --git a/src/modules/data/entities/aggregate/instance/instance.loadout.aggregate.entity.ts b/src/modules/data/entities/aggregate/instance/instance.loadout.aggregate.entity.ts index 0af6c8b7..e979a653 100644 --- a/src/modules/data/entities/aggregate/instance/instance.loadout.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/instance/instance.loadout.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import {Loadout, loadoutArray} from '../../../ps2alerts-constants/loadout'; @@ -15,6 +15,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class InstanceLoadoutAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({example: '10-12345', description: 'The Server-CensusInstanceId combination'}) diff --git a/src/modules/data/entities/aggregate/instance/instance.outfit.aggregate.entity.ts b/src/modules/data/entities/aggregate/instance/instance.outfit.aggregate.entity.ts index 06273218..e1c05c0e 100644 --- a/src/modules/data/entities/aggregate/instance/instance.outfit.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/instance/instance.outfit.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import OutfitEmbed from '../common/outfit.embed'; @@ -16,6 +16,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class InstanceOutfitAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({example: '10-12345', description: 'The Server-CensusInstanceId combination'}) diff --git a/src/modules/data/entities/aggregate/instance/instance.population.aggregate.entity.ts b/src/modules/data/entities/aggregate/instance/instance.population.aggregate.entity.ts index ca9f4744..1bc9c221 100644 --- a/src/modules/data/entities/aggregate/instance/instance.population.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/instance/instance.population.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; @@ -10,6 +10,7 @@ import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; export default class InstancePopulationAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({example: '10-12345', description: 'The Server-CensusInstanceId combination'}) diff --git a/src/modules/data/entities/aggregate/instance/instance.population.averages.aggregate.entity.ts b/src/modules/data/entities/aggregate/instance/instance.population.averages.aggregate.entity.ts index 7cc7332b..9500e48f 100644 --- a/src/modules/data/entities/aggregate/instance/instance.population.averages.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/instance/instance.population.averages.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; @@ -10,6 +10,7 @@ import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; export default class InstancePopulationAveragesAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({example: '10-12345', description: 'The Server-CensusInstanceId combination'}) diff --git a/src/modules/data/entities/aggregate/instance/instance.vehicle.aggregate.entity.ts b/src/modules/data/entities/aggregate/instance/instance.vehicle.aggregate.entity.ts index a59a653e..330c51af 100644 --- a/src/modules/data/entities/aggregate/instance/instance.vehicle.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/instance/instance.vehicle.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import VehicleStatsEmbed from '../common/vehicle.vs.vehicle.embed'; @@ -15,6 +15,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class InstanceVehicleAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({example: '10-12345', description: 'The Server-CensusInstanceId combination'}) diff --git a/src/modules/data/entities/aggregate/instance/instance.vehicle.character.aggregate.entity.ts b/src/modules/data/entities/aggregate/instance/instance.vehicle.character.aggregate.entity.ts index cdcda641..5b67e7a6 100644 --- a/src/modules/data/entities/aggregate/instance/instance.vehicle.character.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/instance/instance.vehicle.character.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import VehicleStatsEmbed from '../common/vehicle.vs.vehicle.embed'; @@ -15,6 +15,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class InstanceVehicleCharacterAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({example: '10-12345', description: 'The Server-CensusInstanceId combination'}) diff --git a/src/modules/data/entities/aggregate/instance/instance.weapon.aggregate.entity.ts b/src/modules/data/entities/aggregate/instance/instance.weapon.aggregate.entity.ts index 3a90a1d1..2ed285ad 100644 --- a/src/modules/data/entities/aggregate/instance/instance.weapon.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/instance/instance.weapon.aggregate.entity.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ -import {ApiProperty} from '@nestjs/swagger'; +import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; import ItemEmbed from '../common/item.embed'; @@ -15,6 +15,7 @@ import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventTyp export default class InstanceWeaponAggregateEntity { @ObjectIdColumn() @Exclude() + @ApiHideProperty() _id: ObjectID; @ApiProperty({example: '10-12345', description: 'The Server-CensusInstanceId combination'}) From 4fc645322105f40dfef2a0cf2b9a1596072fdbea Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Mon, 31 Jul 2023 19:43:45 +0100 Subject: [PATCH 10/15] Added caching around instanceCharacter aggregate --- src/modules/data/ps2alerts-constants | 2 +- .../rest.aggregate.instance.character.controller.ts | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/modules/data/ps2alerts-constants b/src/modules/data/ps2alerts-constants index dacca118..396dbc2d 160000 --- a/src/modules/data/ps2alerts-constants +++ b/src/modules/data/ps2alerts-constants @@ -1 +1 @@ -Subproject commit dacca1186f5df32d8696f94678f6c70d71d711b9 +Subproject commit 396dbc2d58f284d877f7ddb008aa566962ee6794 diff --git a/src/modules/rest/controllers/aggregates/instance/rest.aggregate.instance.character.controller.ts b/src/modules/rest/controllers/aggregates/instance/rest.aggregate.instance.character.controller.ts index 4fc2f43e..98c05133 100644 --- a/src/modules/rest/controllers/aggregates/instance/rest.aggregate.instance.character.controller.ts +++ b/src/modules/rest/controllers/aggregates/instance/rest.aggregate.instance.character.controller.ts @@ -10,12 +10,14 @@ import {Ps2AlertsEventType} from '../../../../data/ps2alerts-constants/ps2Alerts import {AGGREGATE_INSTANCE_COMMON_IMPLICIT_QUERIES} from '../../common/rest.common.queries'; import {PS2ALERTS_EVENT_TYPE_QUERY} from '../../common/rest.ps2AlertsEventType.query'; import {INSTANCE_IMPLICIT_QUERY} from '../../common/rest.instance.query'; +import {RedisCacheService} from '../../../../../services/cache/redis.cache.service'; @ApiTags('Instance Character Aggregates') @Controller('aggregates') export default class RestInstanceCharacterAggregateController { constructor( @Inject(MongoOperationsService) private readonly mongoOperationsService: MongoOperationsService, + private readonly cacheService: RedisCacheService, ) {} @Get('instance/:instance/character') @@ -69,6 +71,13 @@ export default class RestInstanceCharacterAggregateController { @Query('ps2AlertsEventType', Ps2AlertsEventTypePipe) ps2AlertsEventType?: Ps2AlertsEventType, ): Promise { - return await this.mongoOperationsService.findOne(InstanceCharacterAggregateEntity, {'character.id': character, ps2AlertsEventType}); + const key = `cache:instances:instanceCharacter-C:${character}-ET:${ps2AlertsEventType ?? 0}`; + + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return await this.cacheService.get(key) ?? await this.cacheService.set( + key, + await this.mongoOperationsService.findMany(InstanceCharacterAggregateEntity, {'character.id': character, ps2AlertsEventType}), + 60 * 60, + ); } } From 2b0d00abda0937181d44699d51d1a7309e404dc2 Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Mon, 31 Jul 2023 19:44:00 +0100 Subject: [PATCH 11/15] Added missing isArray to GlobalOutfitAggregate --- .../aggregates/global/rest.aggregate.global.outfit.controller.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.outfit.controller.ts b/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.outfit.controller.ts index bed5a315..f1742ba9 100644 --- a/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.outfit.controller.ts +++ b/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.outfit.controller.ts @@ -62,6 +62,7 @@ export default class RestGlobalOutfitAggregateController extends BaseGlobalAggre status: 200, description: 'The GlobalOutfitAggregateEntity aggregate(s)', type: GlobalOutfitAggregateEntity, + isArray: true, }) async findOne( @Param('outfit') outfit: string, From 338aeb564af6d0d24435a6d01b36c5c1c815a0ca Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Mon, 31 Jul 2023 20:32:15 +0100 Subject: [PATCH 12/15] Fixed OptionalBoolPipe --- src/modules/rest/pipes/OptionalBoolPipe.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/rest/pipes/OptionalBoolPipe.ts b/src/modules/rest/pipes/OptionalBoolPipe.ts index 70689e79..ac8b5ca6 100644 --- a/src/modules/rest/pipes/OptionalBoolPipe.ts +++ b/src/modules/rest/pipes/OptionalBoolPipe.ts @@ -2,7 +2,7 @@ import {PipeTransform, Injectable} from '@nestjs/common'; @Injectable() export class OptionalBoolPipe implements PipeTransform { - transform(value: string | undefined): boolean | undefined { - return value === 'true' ?? undefined; + transform(value: string | boolean | undefined): boolean | undefined { + return (value === 'true' || value === true) ?? undefined; } } From e81a751f271b3bdf66d7efa4ef15a498c316ff05 Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Mon, 31 Jul 2023 20:58:04 +0100 Subject: [PATCH 13/15] Added ability to pull in instanceDetails for instance character stuff in order to get victory stats. Made common instance retrieval service to handle caching of instances to make the findMany operation very quick --- .../instance.character.aggregate.entity.ts | 6 +++- ...aggregate.instance.character.controller.ts | 29 +++++++++++++++---- .../rest.instance.metagame.controller.ts | 7 ++--- src/modules/rest/rest.module.ts | 2 ++ src/services/instance.retrieval.service.ts | 26 +++++++++++++++++ 5 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 src/services/instance.retrieval.service.ts diff --git a/src/modules/data/entities/aggregate/instance/instance.character.aggregate.entity.ts b/src/modules/data/entities/aggregate/instance/instance.character.aggregate.entity.ts index 967d89c8..96367d04 100644 --- a/src/modules/data/entities/aggregate/instance/instance.character.aggregate.entity.ts +++ b/src/modules/data/entities/aggregate/instance/instance.character.aggregate.entity.ts @@ -1,11 +1,12 @@ /* eslint-disable @typescript-eslint/explicit-member-accessibility,@typescript-eslint/naming-convention */ import {ApiHideProperty, ApiProperty} from '@nestjs/swagger'; import {Exclude} from 'class-transformer'; -import {Column, ObjectIdColumn, Entity, Index, ObjectID} from 'typeorm'; +import {Column, ObjectIdColumn, Entity, Index, ObjectID, ObjectLiteral} from 'typeorm'; import CharacterEmbed from '../common/character.embed'; import FactionVersusFactionEmbed from '../common/faction.versus.faction.embed'; import XperminuteEmbed from '../common/xperminute.embed'; import {Ps2AlertsEventType} from '../../../ps2alerts-constants/ps2AlertsEventType'; +import InstanceMetagameTerritoryEntity from '../../instance/instance.metagame.territory.entity'; @Entity({ name: 'aggregate_instance_characters', @@ -102,4 +103,7 @@ export default class InstanceCharacterAggregateEntity { default: Ps2AlertsEventType.LIVE_METAGAME, }) ps2AlertsEventType: Ps2AlertsEventType; + + @ApiProperty({type: InstanceMetagameTerritoryEntity, description: 'Instance Metagame Territory'}) + instanceDetails?: ObjectLiteral; } diff --git a/src/modules/rest/controllers/aggregates/instance/rest.aggregate.instance.character.controller.ts b/src/modules/rest/controllers/aggregates/instance/rest.aggregate.instance.character.controller.ts index 98c05133..cba1a456 100644 --- a/src/modules/rest/controllers/aggregates/instance/rest.aggregate.instance.character.controller.ts +++ b/src/modules/rest/controllers/aggregates/instance/rest.aggregate.instance.character.controller.ts @@ -1,6 +1,7 @@ import {Controller, Get, Inject, Param, Query} from '@nestjs/common'; import {ApiOperation, ApiResponse, ApiTags} from '@nestjs/swagger'; -import InstanceCharacterAggregateEntity from '../../../../data/entities/aggregate/instance/instance.character.aggregate.entity'; +import InstanceCharacterAggregateEntity + from '../../../../data/entities/aggregate/instance/instance.character.aggregate.entity'; import MongoOperationsService from '../../../../../services/mongo/mongo.operations.service'; import {OptionalIntPipe} from '../../../pipes/OptionalIntPipe'; import {ApiImplicitQueries} from 'nestjs-swagger-api-implicit-queries-decorator'; @@ -11,6 +12,8 @@ import {AGGREGATE_INSTANCE_COMMON_IMPLICIT_QUERIES} from '../../common/rest.comm import {PS2ALERTS_EVENT_TYPE_QUERY} from '../../common/rest.ps2AlertsEventType.query'; import {INSTANCE_IMPLICIT_QUERY} from '../../common/rest.instance.query'; import {RedisCacheService} from '../../../../../services/cache/redis.cache.service'; +import {OptionalBoolPipe} from '../../../pipes/OptionalBoolPipe'; +import InstanceRetrievalService from '../../../../../services/instance.retrieval.service'; @ApiTags('Instance Character Aggregates') @Controller('aggregates') @@ -18,6 +21,7 @@ export default class RestInstanceCharacterAggregateController { constructor( @Inject(MongoOperationsService) private readonly mongoOperationsService: MongoOperationsService, private readonly cacheService: RedisCacheService, + private readonly instanceRetrievalService: InstanceRetrievalService, ) {} @Get('instance/:instance/character') @@ -66,18 +70,33 @@ export default class RestInstanceCharacterAggregateController { type: InstanceCharacterAggregateEntity, isArray: true, }) - async findByCharacterId( + async findAllByCharacterId( @Param('character') character: string, @Query('ps2AlertsEventType', Ps2AlertsEventTypePipe) ps2AlertsEventType?: Ps2AlertsEventType, - + @Query('getDetails', OptionalBoolPipe) getDetails?: boolean, ): Promise { - const key = `cache:instances:instanceCharacter-C:${character}-ET:${ps2AlertsEventType ?? 0}`; + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + const key = `cache:instance:instanceCharacter:C:${character}-ET:${ps2AlertsEventType ?? 0}-getDetails:${(!!getDetails)}`; // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return await this.cacheService.get(key) ?? await this.cacheService.set( + const alertsInvolved: InstanceCharacterAggregateEntity[] = await this.cacheService.get(key) ?? await this.cacheService.set( key, await this.mongoOperationsService.findMany(InstanceCharacterAggregateEntity, {'character.id': character, ps2AlertsEventType}), 60 * 60, ); + + // If getDetails is set, go off and hydrate the instances with said alerts + if (!getDetails) { + return alertsInvolved; + } + + for (const alert of alertsInvolved) { + // Grab the instance and inject it into the alert + alert.instanceDetails = await this.instanceRetrievalService.findOne(alert.instance); + } + + await this.cacheService.set(key, alertsInvolved, 60 * 60); + + return alertsInvolved; } } diff --git a/src/modules/rest/controllers/rest.instance.metagame.controller.ts b/src/modules/rest/controllers/rest.instance.metagame.controller.ts index 40b68fb2..217bdf67 100644 --- a/src/modules/rest/controllers/rest.instance.metagame.controller.ts +++ b/src/modules/rest/controllers/rest.instance.metagame.controller.ts @@ -39,6 +39,7 @@ import {UpdateInstanceMetagameDto} from '../Dto/UpdateInstanceMetagameDto'; import {CreateInstanceMetagameDto} from '../Dto/CreateInstanceMetagameDto'; import {ObjectID} from 'typeorm'; import {ZONE_IMPLICIT_QUERY} from './common/rest.zone.query'; +import InstanceRetrievalService from '../../../services/instance.retrieval.service'; const INSTANCE_IMPLICIT_QUERIES = [ BRACKET_IMPLICIT_QUERY, @@ -62,6 +63,7 @@ export class RestInstanceMetagameController { constructor( @Inject(MongoOperationsService) private readonly mongoOperationsService: MongoOperationsService, private readonly cacheService: RedisCacheService, + private readonly instanceRetrievalService: InstanceRetrievalService, ) {} @Get('/:instance') @@ -73,10 +75,7 @@ export class RestInstanceMetagameController { }) @UseInterceptors(ClassSerializerInterceptor) async findOne(@Param('instance') instanceId: string): Promise { - return await this.mongoOperationsService.findOne( - InstanceMetagameTerritoryEntity, - {instanceId}, - ); + return await this.instanceRetrievalService.findOne(instanceId); } @Post('') diff --git a/src/modules/rest/rest.module.ts b/src/modules/rest/rest.module.ts index 1589f630..bbac3afd 100644 --- a/src/modules/rest/rest.module.ts +++ b/src/modules/rest/rest.module.ts @@ -60,6 +60,7 @@ import * as redisStore from 'cache-manager-ioredis'; import {RedisCacheService} from '../../services/cache/redis.cache.service'; import {AuthModule} from '../../auth/auth.module'; import RestSearchController from './controllers/rest.search.controller'; +import InstanceRetrievalService from '../../services/instance.retrieval.service'; /** * Handles incoming requests to the API via HTTP, CRUD environment. @@ -143,6 +144,7 @@ import RestSearchController from './controllers/rest.search.controller'; {provide: APP_INTERCEPTOR, useClass: ClassSerializerInterceptor}, MongoOperationsService, RedisCacheService, + InstanceRetrievalService, ], }) export class RestModule {} diff --git a/src/services/instance.retrieval.service.ts b/src/services/instance.retrieval.service.ts new file mode 100644 index 00000000..b9e60db3 --- /dev/null +++ b/src/services/instance.retrieval.service.ts @@ -0,0 +1,26 @@ +import {Inject, Injectable} from '@nestjs/common'; +import MongoOperationsService from './mongo/mongo.operations.service'; +import {RedisCacheService} from './cache/redis.cache.service'; +import InstanceMetagameTerritoryEntity from '../modules/data/entities/instance/instance.metagame.territory.entity'; + +// This service purely grabs the instances out of the database and caches them in a consistent manner. +@Injectable() +export default class InstanceRetrievalService { + constructor( + @Inject(MongoOperationsService) private readonly mongoOperationsService: MongoOperationsService, + private readonly cacheService: RedisCacheService, + ) {} + + public async findOne(instanceId: string): Promise { + const key = `cache:instances:${instanceId}`; + + return await this.cacheService.get(key) ?? await this.cacheService.set( + key, + await this.mongoOperationsService.findOne( + InstanceMetagameTerritoryEntity, + {instanceId}, + ), + 60 * 60 * 24 * 7, + ); + } +} From 06f77947571449589cadc901fed2f523ef4fe21a Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Mon, 31 Jul 2023 21:05:24 +0100 Subject: [PATCH 14/15] Adjusted instance retrieval service to take into account event state and caching policy --- src/modules/data/ps2alerts-constants | 2 +- .../rest.instance.metagame.controller.ts | 4 +-- src/services/instance.retrieval.service.ts | 27 +++++++++++++------ 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/modules/data/ps2alerts-constants b/src/modules/data/ps2alerts-constants index 396dbc2d..4bdfcab5 160000 --- a/src/modules/data/ps2alerts-constants +++ b/src/modules/data/ps2alerts-constants @@ -1 +1 @@ -Subproject commit 396dbc2d58f284d877f7ddb008aa566962ee6794 +Subproject commit 4bdfcab567d392d8f885143b0e7e78e034db5951 diff --git a/src/modules/rest/controllers/rest.instance.metagame.controller.ts b/src/modules/rest/controllers/rest.instance.metagame.controller.ts index 217bdf67..b7c656d4 100644 --- a/src/modules/rest/controllers/rest.instance.metagame.controller.ts +++ b/src/modules/rest/controllers/rest.instance.metagame.controller.ts @@ -37,7 +37,7 @@ import {RedisCacheService} from '../../../services/cache/redis.cache.service'; import {AuthGuard} from '@nestjs/passport'; import {UpdateInstanceMetagameDto} from '../Dto/UpdateInstanceMetagameDto'; import {CreateInstanceMetagameDto} from '../Dto/CreateInstanceMetagameDto'; -import {ObjectID} from 'typeorm'; +import {ObjectID, ObjectLiteral} from 'typeorm'; import {ZONE_IMPLICIT_QUERY} from './common/rest.zone.query'; import InstanceRetrievalService from '../../../services/instance.retrieval.service'; @@ -74,7 +74,7 @@ export class RestInstanceMetagameController { type: InstanceMetagameTerritoryEntity, }) @UseInterceptors(ClassSerializerInterceptor) - async findOne(@Param('instance') instanceId: string): Promise { + async findOne(@Param('instance') instanceId: string): Promise { return await this.instanceRetrievalService.findOne(instanceId); } diff --git a/src/services/instance.retrieval.service.ts b/src/services/instance.retrieval.service.ts index b9e60db3..86607e0a 100644 --- a/src/services/instance.retrieval.service.ts +++ b/src/services/instance.retrieval.service.ts @@ -2,6 +2,8 @@ import {Inject, Injectable} from '@nestjs/common'; import MongoOperationsService from './mongo/mongo.operations.service'; import {RedisCacheService} from './cache/redis.cache.service'; import InstanceMetagameTerritoryEntity from '../modules/data/entities/instance/instance.metagame.territory.entity'; +import {Ps2AlertsEventState} from '../modules/data/ps2alerts-constants/ps2AlertsEventState'; +import {ObjectLiteral} from 'typeorm'; // This service purely grabs the instances out of the database and caches them in a consistent manner. @Injectable() @@ -11,16 +13,25 @@ export default class InstanceRetrievalService { private readonly cacheService: RedisCacheService, ) {} - public async findOne(instanceId: string): Promise { + public async findOne(instanceId: string): Promise { const key = `cache:instances:${instanceId}`; - return await this.cacheService.get(key) ?? await this.cacheService.set( - key, - await this.mongoOperationsService.findOne( - InstanceMetagameTerritoryEntity, - {instanceId}, - ), - 60 * 60 * 24 * 7, + const data = await this.cacheService.get(key); + + if (data) { + return data; + } + + const instance = await this.mongoOperationsService.findOne( + InstanceMetagameTerritoryEntity, + {instanceId}, ); + + // If alert is not complete yet, don't cache it + if (instance.state !== Ps2AlertsEventState.ENDED) { + return instance; + } else { + return await this.cacheService.set(key, instance, 60 * 60 * 24 * 7); + } } } From 5ba70f65c3967e7f829794dd3bd7f02df458c321 Mon Sep 17 00:00:00 2001 From: Matt Cavanagh Date: Fri, 15 Mar 2024 13:53:26 +0000 Subject: [PATCH 15/15] WIP --- .../global/rest.aggregate.global.victory.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.victory.controller.ts b/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.victory.controller.ts index 76812f53..95787def 100644 --- a/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.victory.controller.ts +++ b/src/modules/rest/controllers/aggregates/global/rest.aggregate.global.victory.controller.ts @@ -65,7 +65,7 @@ export default class RestGlobalVictoryAggregateController extends BaseGlobalAggr }; const key = `cache:endpoints:victories:W:${world ?? 0}-Z:${zone ?? 0}-B:${bracket ?? 0}-ET:${ps2AlertsEventType ?? 0}-DF:${dateFrom ? dateFrom.toString() : 0}-DT:${dateTo ? dateTo.toString() : 0}`; - const pagination = new Pagination({sortBy: 'date', order: 'desc'}); + const pagination = new Pagination({sortBy: 'date', order: 'asc'}); // eslint-disable-next-line @typescript-eslint/no-unsafe-return return await this.cacheService.get(key) ?? await this.cacheService.set(