From 7d86ea3d92d34471d3ad81e65f01151e29067fa9 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Mon, 2 Oct 2023 12:32:34 -0400 Subject: [PATCH] make wwp more accurate and stable --- package-lock.json | 130 ----------------- package.json | 2 - src/cache.ts | 27 ++++ src/middleware/auth.ts | 2 + src/models/post.ts | 55 ++++++-- src/services/api/routes/topics.ts | 208 +++++++++++++++++++++------- src/types/common/post.ts | 11 ++ src/types/common/wara-wara-plaza.ts | 70 ++++++++++ src/types/memoizee.d.ts | 34 ----- src/types/mongoose/post.ts | 7 + 10 files changed, 316 insertions(+), 230 deletions(-) create mode 100644 src/cache.ts create mode 100644 src/types/common/post.ts create mode 100644 src/types/common/wara-wara-plaza.ts delete mode 100644 src/types/memoizee.d.ts diff --git a/package-lock.json b/package-lock.json index c37b27a..9bea127 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,6 @@ "express": "^4.17.1", "express-subdomain": "^1.0.5", "fs-extra": "^9.0.0", - "memoizee": "^0.4.15", "moment": "^2.24.0", "mongoose": "^6.10.1", "morgan": "^1.10.0", @@ -34,7 +33,6 @@ "devDependencies": { "@types/express": "^4.17.17", "@types/fs-extra": "^11.0.1", - "@types/memoizee": "^0.4.8", "@types/morgan": "^1.9.4", "@types/multer": "^1.4.7", "@types/newman": "^5.3.2", @@ -1488,12 +1486,6 @@ "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" }, - "node_modules/@types/memoizee": { - "version": "0.4.8", - "resolved": "https://registry.npmjs.org/@types/memoizee/-/memoizee-0.4.8.tgz", - "integrity": "sha512-qDpXKGgwKywnQt/64fH1O0LiPA++QGIYeykEUiZ51HymKVRLnUSGcRuF60IfpPeeXiuRwiR/W4y7S5VzbrgLCA==", - "dev": true - }, "node_modules/@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -2635,15 +2627,6 @@ "integrity": "sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==", "dev": true }, - "node_modules/d": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", - "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", - "dependencies": { - "es5-ext": "^0.10.50", - "type": "^1.0.1" - } - }, "node_modules/dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -2776,50 +2759,6 @@ "node": ">= 0.8" } }, - "node_modules/es5-ext": { - "version": "0.10.62", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.62.tgz", - "integrity": "sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==", - "hasInstallScript": true, - "dependencies": { - "es6-iterator": "^2.0.3", - "es6-symbol": "^3.1.3", - "next-tick": "^1.1.0" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/es6-iterator": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", - "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==", - "dependencies": { - "d": "1", - "es5-ext": "^0.10.35", - "es6-symbol": "^3.1.1" - } - }, - "node_modules/es6-symbol": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", - "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", - "dependencies": { - "d": "^1.0.1", - "ext": "^1.1.2" - } - }, - "node_modules/es6-weak-map": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.3.tgz", - "integrity": "sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==", - "dependencies": { - "d": "1", - "es5-ext": "^0.10.46", - "es6-iterator": "^2.0.3", - "es6-symbol": "^3.1.1" - } - }, "node_modules/escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -3050,15 +2989,6 @@ "node": ">= 0.6" } }, - "node_modules/event-emitter": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", - "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==", - "dependencies": { - "d": "1", - "es5-ext": "~0.10.14" - } - }, "node_modules/eventemitter3": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", @@ -3132,19 +3062,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, - "node_modules/ext": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", - "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==", - "dependencies": { - "type": "^2.7.2" - } - }, - "node_modules/ext/node_modules/type": { - "version": "2.7.2", - "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz", - "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==" - }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -3910,11 +3827,6 @@ "node": ">=8" } }, - "node_modules/is-promise": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz", - "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==" - }, "node_modules/is-typed-array": { "version": "1.1.10", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.10.tgz", @@ -4173,14 +4085,6 @@ "node": ">=10" } }, - "node_modules/lru-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz", - "integrity": "sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ==", - "dependencies": { - "es5-ext": "~0.10.2" - } - }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -4189,21 +4093,6 @@ "node": ">= 0.6" } }, - "node_modules/memoizee": { - "version": "0.4.15", - "resolved": "https://registry.npmjs.org/memoizee/-/memoizee-0.4.15.tgz", - "integrity": "sha512-UBWmJpLZd5STPm7PMUlOw/TSy972M+z8gcyQ5veOnSDRREz/0bmpyTfKt3/51DhEBqCZQn1udM/5flcSPYhkdQ==", - "dependencies": { - "d": "^1.0.1", - "es5-ext": "^0.10.53", - "es6-weak-map": "^2.0.3", - "event-emitter": "^0.3.5", - "is-promise": "^2.2.2", - "lru-queue": "^0.1.0", - "next-tick": "^1.1.0", - "timers-ext": "^0.1.7" - } - }, "node_modules/memory-pager": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", @@ -4629,11 +4518,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/next-tick": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", - "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==" - }, "node_modules/nice-grpc": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/nice-grpc/-/nice-grpc-2.1.4.tgz", @@ -6003,15 +5887,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, - "node_modules/timers-ext": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz", - "integrity": "sha512-b85NUNzTSdodShTIbky6ZF02e8STtVVfD+fu4aXXShEELpozH+bCpJLYMPZbsABN2wDH7fJpqIoXxJpzbf0NqQ==", - "dependencies": { - "es5-ext": "~0.10.46", - "next-tick": "1" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -6157,11 +6032,6 @@ "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", "dev": true }, - "node_modules/type": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", - "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" - }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index b709018..35c3be9 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,6 @@ "express": "^4.17.1", "express-subdomain": "^1.0.5", "fs-extra": "^9.0.0", - "memoizee": "^0.4.15", "moment": "^2.24.0", "mongoose": "^6.10.1", "morgan": "^1.10.0", @@ -39,7 +38,6 @@ "devDependencies": { "@types/express": "^4.17.17", "@types/fs-extra": "^11.0.1", - "@types/memoizee": "^0.4.8", "@types/morgan": "^1.9.4", "@types/multer": "^1.4.7", "@types/newman": "^5.3.2", diff --git a/src/cache.ts b/src/cache.ts new file mode 100644 index 0000000..e8f9743 --- /dev/null +++ b/src/cache.ts @@ -0,0 +1,27 @@ +export default class Cache { + private data?: T; + private expireAt: number; + private cacheTime: number; + + constructor(cacheTime: number) { + this.expireAt = Date.now() + cacheTime; + this.cacheTime = cacheTime; + } + + valid(): boolean { + if (!this.data || Date.now() >= this.expireAt) { + return false; + } + + return true; + } + + update(data: T): void { + this.expireAt = Date.now() + this.cacheTime; + this.data = data; + } + + get(): T | undefined { + return this.data; + } +} \ No newline at end of file diff --git a/src/middleware/auth.ts b/src/middleware/auth.ts index bb3f777..fb7f563 100644 --- a/src/middleware/auth.ts +++ b/src/middleware/auth.ts @@ -52,6 +52,7 @@ async function auth(request: express.Request, response: express.Response, next: const paramPackData: ParamPack = decodeParamPack(paramPack); const paramPackCheck: z.SafeParseReturnType = ParamPackSchema.safeParse(paramPackData); if (!paramPackCheck.success) { + console.log(paramPackCheck.error); return badAuth(response, 18, 'BAD_PARAM'); } @@ -61,6 +62,7 @@ async function auth(request: express.Request, response: express.Response, next: user = await getUserAccountData(pid); } catch (error) { // TODO - Log this error + console.log(error); return badAuth(response, 18, 'BAD_PARAM'); } diff --git a/src/models/post.ts b/src/models/post.ts index d4c8728..ead834f 100644 --- a/src/models/post.ts +++ b/src/models/post.ts @@ -4,6 +4,7 @@ import { Schema, model } from 'mongoose'; import { HydratedPostDocument, IPost, IPostMethods, PostModel } from '@/types/mongoose/post'; import { HydratedCommunityDocument } from '@/types/mongoose/community'; import { PostToJSONOptions } from '@/types/mongoose/post-to-json-options'; +import { PostPainting, PostScreenshot } from '@/types/common/post'; const PostSchema = new Schema({ id: String, @@ -132,9 +133,45 @@ PostSchema.method('generatePostUID', async function generatePostUID(length: numb } }); +PostSchema.method('cleanedBody', function cleanedBody(): string { + return this.body ? this.body.replace(/[^A-Za-z\d\s-_!@#$%^&*(){}+=,.<>/?;:'"[\]]/g, '').replace(/[\n\r]+/gm, '') : ''; +}); + +PostSchema.method('cleanedMiiData', function cleanedMiiData(): string { + return this.mii.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim(); +}); + +PostSchema.method('cleanedPainting', function cleanedPainting(): string { + return this.painting.replace(/[\n\r]+/gm, '').trim(); +}); + +PostSchema.method('cleanedAppData', function cleanedAppData(): string { + return this.app_data.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim(); +}); + +PostSchema.method('formatPainting', function formatPainting(): PostPainting | undefined { + if (this.painting) { + return { + format: 'tga', + content: this.cleanedPainting(), + size: this.painting.length, + url: `https://pretendo-cdn.b-cdn.net/paintings/${this.pid}/${this.id}.png` + }; + } +}); + +PostSchema.method('formatScreenshot', function formatScreenshot(): PostScreenshot | undefined { + if (this.screenshot && this.screenshot_length) { + return { + size: this.screenshot_length, + url: `https://pretendo-cdn.b-cdn.net/screenshots/${this.pid}/${this.id}.jpg` + }; + } +}); + PostSchema.method('json', function json(options: PostToJSONOptions, community?: HydratedCommunityDocument): Record { const json: Record = { - body: this.body ? this.body.replace(/[^A-Za-z\d\s-_!@#$%^&*(){}+=,.<>/?;:'"[\]]/g, '').replace(/[\n\r]+/gm, '') : '', + body: this.cleanedBody(), country_id: this.country_id ? this.country_id : 254, created_at: moment(this.created_at).format('YYYY-MM-DD HH:MM:SS'), feeling_id: this.feeling_id, @@ -155,7 +192,7 @@ PostSchema.method('json', function json(options: PostToJSONOptions, community?: }; if (this.app_data && options.app_data) { - json.app_data = this.app_data.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim(); + json.app_data = this.cleanedAppData(); } if (options.topics && community) { @@ -165,24 +202,16 @@ PostSchema.method('json', function json(options: PostToJSONOptions, community?: } if (options.with_mii) { - json.mii = this.mii.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim(); + json.mii = this.cleanedMiiData(); json.mii_face_url = this.mii_face_url; } if (this.painting) { - json.painting = { - format: 'tga', - content: this.painting.replace(/[\n\r]+/gm, '').trim(), - size: this.painting.length, - url: `https://pretendo-cdn.b-cdn.net/paintings/${this.pid}/${this.id}.png` - }; + json.painting = this.formatPainting(); } if (this.screenshot && this.screenshot_length) { - json.screenshot = { - size: this.screenshot_length, - url: `https://pretendo-cdn.b-cdn.net/screenshots/${this.pid}/${this.id}.jpg` - }; + json.screenshot = this.formatScreenshot(); } if (this.topic_tag && options.topic_tag) { diff --git a/src/services/api/routes/topics.ts b/src/services/api/routes/topics.ts index d27401c..c5d8ce2 100644 --- a/src/services/api/routes/topics.ts +++ b/src/services/api/routes/topics.ts @@ -1,23 +1,19 @@ import express from 'express'; -import memoize from 'memoizee'; import moment from 'moment'; import xmlbuilder from 'xmlbuilder'; import { GetUserDataResponse } from 'pretendo-grpc-ts/dist/account/get_user_data_rpc'; import { getUserAccountData } from '@/util'; -import { getEndpoint, getPostsBytitleID } from '@/database'; +import Cache from '@/cache'; +import { getEndpoint } from '@/database'; import { Post } from '@/models/post'; import { Community } from '@/models/community'; import { HydratedEndpointDocument } from '@/types/mongoose/endpoint'; import { HydratedCommunityDocument } from '@/types/mongoose/community'; -import { HydratedPostDocument } from '@/types/mongoose/post'; +import { WWPData, WWPPost, WWPTopic } from '@/types/common/wara-wara-plaza'; -const router: express.Router = express.Router(); - -// TODO - Need to add types to memoize in @/types/memoize.d.ts -const memoizedGenerateTopicsXML = memoize(generateTopicsXML, { - async: true, - maxAge: 1000 * 60 * 60 // * cache for 1 hour -}); +const router = express.Router(); +const ONE_HOUR = 60 * 60 * 1000; +const WARA_WARA_PLAZA_CACHE = new Cache(ONE_HOUR); /* GET post titles. */ router.get('/', async function (request: express.Request, response: express.Response): Promise { @@ -46,77 +42,184 @@ router.get('/', async function (request: express.Request, response: express.Resp return; } - const communities: HydratedCommunityDocument[] = await calculateMostPopularCommunities(24, 10); + if (!WARA_WARA_PLAZA_CACHE.valid()) { + const communities: HydratedCommunityDocument[] = await calculateMostPopularCommunities(24, 10); - if (communities.length < 10) { - response.sendStatus(404); - return; + if (communities.length < 10) { + response.sendStatus(404); + return; + } + + WARA_WARA_PLAZA_CACHE.update(await generateTopicsData(communities)); } - response.send(await memoizedGenerateTopicsXML(communities)); + const data = WARA_WARA_PLAZA_CACHE.get() || {}; + + response.send(xmlbuilder.create(data, { separateArrayItems: true }).end({ pretty: true, allowEmpty: true })); }); -async function generateTopicsXML(communities: HydratedCommunityDocument[]): Promise { - const json: Record = { - result: { - has_error: 0, - version: 1, - expire: moment().add(2, 'days').format('YYYY-MM-DD HH:MM:SS'), - request_name: 'topics', - topics: [] - } - }; +async function generateTopicsData(communities: HydratedCommunityDocument[]): Promise { + const topics: { + topic: WWPTopic; + }[] = []; - for (const community of communities) { - const topic: Record = { - topic: { - empathy_count: community.empathy_count, - has_shop_page: community.has_shop_page, - icon: community.icon, - title_ids: [], - title_id: community.title_id[0], - community_id: community.community_id, - is_recommended: community.is_recommended, - name: community.name, - people: [] + for (let i = 0; i < communities.length; i++) { + const community = communities[i]; + + const empathies = await Post.aggregate([ + { + $match: { + community_id: community.olive_community_id + } + }, + { + $group : { + _id : null, + total : { + $sum : '$empathy_count' + } + } + }, + { + $limit: 1 } + ]); + + const topic: WWPTopic = { + empathy_count: empathies[0]?.total || 0, + has_shop_page: community.has_shop_page ? 1 : 0, + icon: community.icon, + title_ids: [], + title_id: community.title_id[0], + community_id: 0xFFFFFFFF, // * This is how it was in the real WWP. Unsure why, but it works + is_recommended: community.is_recommended ? 1 : 0, + name: community.name, + people: [], + position: i+1 }; - community.title_id.forEach(function (title_id: string) { - if (title_id !== '') { - topic.topic.title_ids.push({ title_id }); + community.title_id.forEach(title_id => { + // * Just in case + if (title_id) { + topic.title_ids.push({ title_id }); } }); - const posts: HydratedPostDocument[] = await getPostsBytitleID(community.title_id, 30); + const people = await getCommunityPeople(community); - for (const post of posts) { - topic.topic.people.push({ + for (const person of people) { + const hydratedPost = Post.hydrate(person.post); + + const post: WWPPost = { + body: hydratedPost.cleanedBody(), + community_id: 0xFFFFFFFF, // * This is how it was in the real WWP. Unsure why, but it works + country_id: hydratedPost.country_id, + created_at: moment(hydratedPost.created_at).format('YYYY-MM-DD HH:MM:SS'), + feeling_id: hydratedPost.feeling_id, + id: hydratedPost.id, + is_autopost: hydratedPost.is_autopost ? 1 : 0, + is_community_private_autopost: hydratedPost.is_community_private_autopost ? 1 : 0, + is_spoiler: hydratedPost.is_spoiler ? 1 : 0, + is_app_jumpable: hydratedPost.is_app_jumpable ? 1 : 0, + empathy_count: hydratedPost.empathy_count || 0, + language_id: hydratedPost.language_id, + mii: hydratedPost.cleanedMiiData(), + mii_face_url: hydratedPost.mii_face_url, + number: 0, + painting: hydratedPost.formatPainting(), + pid: hydratedPost.pid, + platform_id: hydratedPost.platform_id, + region_id: hydratedPost.region_id, + reply_count: hydratedPost.reply_count || 0, + screen_name: hydratedPost.screen_name, + screenshot: hydratedPost.formatScreenshot(), + title_id: hydratedPost.title_id, + }; + + topic.people.push({ person: { posts: [ { - post: post.json({ - with_mii: true, - topics: true - }) + post } ] } }); } - json.result.topics.push(topic); + topics.push({ + topic: topic + }); } - return xmlbuilder.create(json, { separateArrayItems: true }).end({ pretty: true, allowEmpty: true }); + return { + result: { + has_error: 0, + version: 1, + expire: moment().add(2, 'days').format('YYYY-MM-DD HH:MM:SS'), + request_name: 'topics', + topics + } + }; +} + +async function getCommunityPeople(community: HydratedCommunityDocument, hours = 24): Promise { + const now = new Date(); + const last24Hours = new Date(now.getTime() - hours * 60 * 60 * 1000); + const people = await Post.aggregate([ + { + $match: { + title_id: { + $in: community.title_id + }, + created_at: { + $gte: last24Hours + }, + message_to_pid: null, + parent: null, + removed: false + } + }, + { + $group: { + _id: '$pid', + post: { + $first: '$$ROOT' + } + } + }, + { + $limit: 70 // * Arbitrary + } + ]); + + // TODO - Remove this check once out of beta and have more users + // * We only do this because Juxtaposition is not super active + // * due to it being in beta. If we don't expand the search + // * time range then WWP still ends up fairly empty + // * + // * Ensure we have at *least* 20 people. Arbitrary. + // * If the year is less than 2020, assume we've gone + // * too far back. There are no more posts, just return + // * what was found + if (people.length < 20 && last24Hours.getFullYear() >= 2020) { + // * Double the search range each time to get + // * exponentially more posts. This speeds up + // * the search at the cost of using older posts + return getCommunityPeople(community, hours * 2); + } + + return people; } async function calculateMostPopularCommunities(hours: number, limit: number): Promise { - const now: Date = new Date(); - const last24Hours: Date = new Date(now.getTime() - hours * 60 * 60 * 1000); + const now = new Date(); + const last24Hours = new Date(now.getTime() - hours * 60 * 60 * 1000); + if (!last24Hours) { throw new Error('Invalid date'); } + const validCommunities: { _id: null; communities: [string]; @@ -136,10 +239,13 @@ async function calculateMostPopularCommunities(hours: number, limit: number): Pr } } ]); + const communityIDs: [string] = validCommunities[0].communities; + if (!communityIDs) { throw new Error('No communities found'); } + const popularCommunities: { _id: string; count: number; diff --git a/src/types/common/post.ts b/src/types/common/post.ts new file mode 100644 index 0000000..2807c31 --- /dev/null +++ b/src/types/common/post.ts @@ -0,0 +1,11 @@ +export type PostPainting = { + format: string; + content: string; + size: number; + url: string; +}; + +export type PostScreenshot = { + size: number; + url: string; +}; \ No newline at end of file diff --git a/src/types/common/wara-wara-plaza.ts b/src/types/common/wara-wara-plaza.ts new file mode 100644 index 0000000..5454598 --- /dev/null +++ b/src/types/common/wara-wara-plaza.ts @@ -0,0 +1,70 @@ +// TODO - Maybe this can become more generalized, and not specific to WWP? +export type WWPPost = { + body?: string; + community_id: number; + country_id: number; + created_at: string; + feeling_id: number; + id: string; + is_autopost: 0 | 1; + is_community_private_autopost: 0 | 1; + is_spoiler: 0 | 1; + is_app_jumpable: 0 | 1; + empathy_count: number; + language_id: number; + mii: string; + mii_face_url: string; + number: number; + painting?: { + format: string; + content: string; + size: number; + url: string; + }; + pid: number; + platform_id: number; + region_id: number; + reply_count: number; + screen_name: string; + screenshot?: { + size: number; + url: string; + }; + title_id: string; +}; + +export type WWPPerson = { + person: { + posts: { + post: WWPPost; + }[]; + } +}; + +export type WWPTopic = { + empathy_count: number; + has_shop_page: 0 | 1; + icon: string; + title_ids: { + title_id: string; + }[]; + title_id: string; + community_id: number; + is_recommended: 0 | 1; + name: string; + people: WWPPerson[]; + position: number; +}; + +export type WWPData = { + result: { + has_error: 0 | 1; + version: 1; + expire: string; + request_name: 'topics'; + topics: { + topic: WWPTopic; + }[]; + } +}; + diff --git a/src/types/memoizee.d.ts b/src/types/memoizee.d.ts deleted file mode 100644 index ad4f77b..0000000 --- a/src/types/memoizee.d.ts +++ /dev/null @@ -1,34 +0,0 @@ -// * Taken from https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/memoizee/index.d.ts -// * Installing from @types/memoizee did not work, types werent being seen - -// Type definitions for memoizee 0.4 -// Project: https://github.com/medikoo/memoizee -// Definitions by: Juan Picado -// Patrick Muff -// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped -// Minimum TypeScript Version: 3.1 - -declare namespace memoizee { - interface Options any> { - length?: number | false | undefined; - maxAge?: number | undefined; - max?: number | undefined; - preFetch?: number | true | undefined; - promise?: boolean | 'then' | 'done' | 'done:finally' | undefined; - dispose?(value: any): void; - async?: boolean | undefined; - primitive?: boolean | undefined; - normalizer?(args: Parameters): string; - resolvers?: Array<(arg: any) => any> | undefined; - profileName?: string; - } - - interface Memoized { - delete: F; - clear: F & (() => void); - } -} - -declare function memoizee any>(f: F, options?: memoizee.Options): F & memoizee.Memoized; - -export = memoizee; \ No newline at end of file diff --git a/src/types/mongoose/post.ts b/src/types/mongoose/post.ts index 0b5c286..6e83992 100644 --- a/src/types/mongoose/post.ts +++ b/src/types/mongoose/post.ts @@ -1,6 +1,7 @@ import { Model, Types, HydratedDocument } from 'mongoose'; import { HydratedCommunityDocument } from '@/types/mongoose/community'; import { PostToJSONOptions } from '@/types/mongoose/post-to-json-options'; +import { PostPainting, PostScreenshot } from '@/types/common/post'; export interface IPost { id: string; @@ -44,6 +45,12 @@ export interface IPostMethods { remove(reason: string): Promise; unRemove(reason: string): Promise; generatePostUID(length: number): Promise; + cleanedBody(): string; + cleanedMiiData(): string; + cleanedPainting(): string; + cleanedAppData(): string; + formatPainting(): PostPainting | undefined; + formatScreenshot(): PostScreenshot | undefined; json(options: PostToJSONOptions, community?: HydratedCommunityDocument): Record; }