make wwp more accurate and stable

src/cache.ts

@@ -0,0 +1,27 @@
+export default class Cache<T> {
+	private data?: T;
+	private expireAt: number;
+	private cacheTime: number;
+
+	constructor(cacheTime: number) {
+		this.expireAt = Date.now() + cacheTime;
+		this.cacheTime = cacheTime;
+	}
+
+	valid(): boolean {
+		if (!this.data || Date.now() >= this.expireAt) {
+			return false;
+		}
+
+		return true;
+	}
+
+	update(data: T): void {
+		this.expireAt = Date.now() + this.cacheTime;
+		this.data = data;
+	}
+
+	get(): T | undefined {
+		return this.data;
+	}
+}

src/middleware/auth.ts

@@ -52,6 +52,7 @@ async function auth(request: express.Request, response: express.Response, next:
 	const paramPackData: ParamPack = decodeParamPack(paramPack);
 	const paramPackCheck: z.SafeParseReturnType<ParamPack, ParamPack> = ParamPackSchema.safeParse(paramPackData);
 	if (!paramPackCheck.success) {
+		console.log(paramPackCheck.error);
 		return badAuth(response, 18, 'BAD_PARAM');
 	}
 
@@ -61,6 +62,7 @@ async function auth(request: express.Request, response: express.Response, next:
 		user = await getUserAccountData(pid);
 	} catch (error) {
 		// TODO - Log this error
+		console.log(error);
 		return badAuth(response, 18, 'BAD_PARAM');
 	}
 

src/models/post.ts

@@ -4,6 +4,7 @@ import { Schema, model } from 'mongoose';
 import { HydratedPostDocument, IPost, IPostMethods, PostModel } from '@/types/mongoose/post';
 import { HydratedCommunityDocument } from '@/types/mongoose/community';
 import { PostToJSONOptions } from '@/types/mongoose/post-to-json-options';
+import { PostPainting, PostScreenshot } from '@/types/common/post';
 
 const PostSchema = new Schema<IPost, PostModel, IPostMethods>({
 	id: String,
@@ -132,9 +133,45 @@ PostSchema.method('generatePostUID', async function generatePostUID(length: numb
 	}
 });
 
+PostSchema.method('cleanedBody', function cleanedBody(): string {
+	return this.body ? this.body.replace(/[^A-Za-z\d\s-_!@#$%^&*(){}+=,.<>/?;:'"[\]]/g, '').replace(/[\n\r]+/gm, '') : '';
+});
+
+PostSchema.method('cleanedMiiData', function cleanedMiiData(): string {
+	return this.mii.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim();
+});
+
+PostSchema.method('cleanedPainting', function cleanedPainting(): string {
+	return this.painting.replace(/[\n\r]+/gm, '').trim();
+});
+
+PostSchema.method('cleanedAppData', function cleanedAppData(): string {
+	return this.app_data.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim();
+});
+
+PostSchema.method('formatPainting', function formatPainting(): PostPainting | undefined {
+	if (this.painting) {
+		return {
+			format: 'tga',
+			content: this.cleanedPainting(),
+			size: this.painting.length,
+			url: `https://pretendo-cdn.b-cdn.net/paintings/${this.pid}/${this.id}.png`
+		};
+	}
+});
+
+PostSchema.method('formatScreenshot', function formatScreenshot(): PostScreenshot | undefined {
+	if (this.screenshot && this.screenshot_length) {
+		return {
+			size: this.screenshot_length,
+			url: `https://pretendo-cdn.b-cdn.net/screenshots/${this.pid}/${this.id}.jpg`
+		};
+	}
+});
+
 PostSchema.method('json', function json(options: PostToJSONOptions, community?: HydratedCommunityDocument): Record<string, any> {
 	const json: Record<string, any> = {
-		body: this.body ? this.body.replace(/[^A-Za-z\d\s-_!@#$%^&*(){}+=,.<>/?;:'"[\]]/g, '').replace(/[\n\r]+/gm, '') : '',
+		body: this.cleanedBody(),
 		country_id: this.country_id ? this.country_id : 254,
 		created_at: moment(this.created_at).format('YYYY-MM-DD HH:MM:SS'),
 		feeling_id: this.feeling_id,
@@ -155,7 +192,7 @@ PostSchema.method('json', function json(options: PostToJSONOptions, community?:
 	};
 
 	if (this.app_data && options.app_data) {
-		json.app_data = this.app_data.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim();
+		json.app_data = this.cleanedAppData();
 	}
 
 	if (options.topics && community) {
@@ -165,24 +202,16 @@ PostSchema.method('json', function json(options: PostToJSONOptions, community?:
 	}
 
 	if (options.with_mii) {
-		json.mii = this.mii.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim();
+		json.mii = this.cleanedMiiData();
 		json.mii_face_url = this.mii_face_url;
 	}
 
 	if (this.painting) {
-		json.painting = {
-			format: 'tga',
-			content: this.painting.replace(/[\n\r]+/gm, '').trim(),
-			size: this.painting.length,
-			url: `https://pretendo-cdn.b-cdn.net/paintings/${this.pid}/${this.id}.png`
-		};
+		json.painting = this.formatPainting();
 	}
 
 	if (this.screenshot && this.screenshot_length) {
-		json.screenshot = {
-			size: this.screenshot_length,
-			url: `https://pretendo-cdn.b-cdn.net/screenshots/${this.pid}/${this.id}.jpg`
-		};
+		json.screenshot = this.formatScreenshot();
 	}
 
 	if (this.topic_tag && options.topic_tag) {

src/services/api/routes/topics.ts

@@ -1,23 +1,19 @@
 import express from 'express';
-import memoize from 'memoizee';
 import moment from 'moment';
 import xmlbuilder from 'xmlbuilder';
 import { GetUserDataResponse } from 'pretendo-grpc-ts/dist/account/get_user_data_rpc';
 import { getUserAccountData } from '@/util';
-import { getEndpoint, getPostsBytitleID } from '@/database';
+import Cache from '@/cache';
+import { getEndpoint } from '@/database';
 import { Post } from '@/models/post';
 import { Community } from '@/models/community';
 import { HydratedEndpointDocument } from '@/types/mongoose/endpoint';
 import { HydratedCommunityDocument } from '@/types/mongoose/community';
-import { HydratedPostDocument } from '@/types/mongoose/post';
+import { WWPData, WWPPost, WWPTopic } from '@/types/common/wara-wara-plaza';
 
-const router: express.Router = express.Router();
-
-// TODO - Need to add types to memoize in @/types/memoize.d.ts
-const memoizedGenerateTopicsXML = memoize(generateTopicsXML, {
-	async: true,
-	maxAge: 1000 * 60 * 60 // * cache for 1 hour
-});
+const router = express.Router();
+const ONE_HOUR = 60 * 60 * 1000;
+const WARA_WARA_PLAZA_CACHE = new Cache<WWPData>(ONE_HOUR);
 
 /* GET post titles. */
 router.get('/', async function (request: express.Request, response: express.Response): Promise<void> {
@@ -46,77 +42,184 @@ router.get('/', async function (request: express.Request, response: express.Resp
 		return;
 	}
 
-	const communities: HydratedCommunityDocument[] = await calculateMostPopularCommunities(24, 10);
+	if (!WARA_WARA_PLAZA_CACHE.valid()) {
+		const communities: HydratedCommunityDocument[] = await calculateMostPopularCommunities(24, 10);
 
-	if (communities.length < 10) {
-		response.sendStatus(404);
-		return;
+		if (communities.length < 10) {
+			response.sendStatus(404);
+			return;
+		}
+
+		WARA_WARA_PLAZA_CACHE.update(await generateTopicsData(communities));
 	}
 
-	response.send(await memoizedGenerateTopicsXML(communities));
+	const data = WARA_WARA_PLAZA_CACHE.get() || {};
+
+	response.send(xmlbuilder.create(data, { separateArrayItems: true }).end({ pretty: true, allowEmpty: true }));
 });
 
-async function generateTopicsXML(communities: HydratedCommunityDocument[]): Promise<string> {
-	const json: Record<string, any> = {
-		result: {
-			has_error: 0,
-			version: 1,
-			expire: moment().add(2, 'days').format('YYYY-MM-DD HH:MM:SS'),
-			request_name: 'topics',
-			topics: []
-		}
-	};
+async function generateTopicsData(communities: HydratedCommunityDocument[]): Promise<WWPData> {
+	const topics: {
+		topic: WWPTopic;
+	}[] = [];
 
-	for (const community of communities) {
-		const topic: Record<string, any> = {
-			topic: {
-				empathy_count: community.empathy_count,
-				has_shop_page: community.has_shop_page,
-				icon: community.icon,
-				title_ids: [],
-				title_id: community.title_id[0],
-				community_id: community.community_id,
-				is_recommended: community.is_recommended,
-				name: community.name,
-				people: []
+	for (let i = 0; i < communities.length; i++) {
+		const community = communities[i];
+
+		const empathies = await Post.aggregate([
+			{
+				$match: {
+					community_id: community.olive_community_id
+				}
+			},
+			{
+				$group : {
+					_id : null,
+					total : {
+						$sum : '$empathy_count'
+					}
+				}
+			},
+			{
+				$limit: 1
 			}
+		]);
+
+		const topic: WWPTopic = {
+			empathy_count: empathies[0]?.total || 0,
+			has_shop_page: community.has_shop_page ? 1 : 0,
+			icon: community.icon,
+			title_ids: [],
+			title_id: community.title_id[0],
+			community_id: 0xFFFFFFFF, // * This is how it was in the real WWP. Unsure why, but it works
+			is_recommended: community.is_recommended ? 1 : 0,
+			name: community.name,
+			people: [],
+			position: i+1
 		};
 
-		community.title_id.forEach(function (title_id: string) {
-			if (title_id !== '') {
-				topic.topic.title_ids.push({ title_id });
+		community.title_id.forEach(title_id => {
+			// * Just in case
+			if (title_id) {
+				topic.title_ids.push({ title_id });
 			}
 		});
 
-		const posts: HydratedPostDocument[] = await getPostsBytitleID(community.title_id, 30);
+		const people = await getCommunityPeople(community);
 
-		for (const post of posts) {
-			topic.topic.people.push({
+		for (const person of people) {
+			const hydratedPost = Post.hydrate(person.post);
+
+			const post: WWPPost = {
+				body: hydratedPost.cleanedBody(),
+				community_id: 0xFFFFFFFF, // * This is how it was in the real WWP. Unsure why, but it works
+				country_id: hydratedPost.country_id,
+				created_at: moment(hydratedPost.created_at).format('YYYY-MM-DD HH:MM:SS'),
+				feeling_id: hydratedPost.feeling_id,
+				id: hydratedPost.id,
+				is_autopost: hydratedPost.is_autopost ? 1 : 0,
+				is_community_private_autopost: hydratedPost.is_community_private_autopost ? 1 : 0,
+				is_spoiler: hydratedPost.is_spoiler ? 1 : 0,
+				is_app_jumpable: hydratedPost.is_app_jumpable ? 1 : 0,
+				empathy_count: hydratedPost.empathy_count || 0,
+				language_id: hydratedPost.language_id,
+				mii: hydratedPost.cleanedMiiData(),
+				mii_face_url: hydratedPost.mii_face_url,
+				number: 0,
+				painting: hydratedPost.formatPainting(),
+				pid: hydratedPost.pid,
+				platform_id: hydratedPost.platform_id,
+				region_id: hydratedPost.region_id,
+				reply_count: hydratedPost.reply_count || 0,
+				screen_name: hydratedPost.screen_name,
+				screenshot: hydratedPost.formatScreenshot(),
+				title_id: hydratedPost.title_id,
+			};
+
+			topic.people.push({
 				person: {
 					posts: [
 						{
-							post: post.json({
-								with_mii: true,
-								topics: true
-							})
+							post
 						}
 					]
 				}
 			});
 		}
 
-		json.result.topics.push(topic);
+		topics.push({
+			topic: topic
+		});
 	}
 
-	return xmlbuilder.create(json, { separateArrayItems: true }).end({ pretty: true, allowEmpty: true });
+	return {
+		result: {
+			has_error: 0,
+			version: 1,
+			expire: moment().add(2, 'days').format('YYYY-MM-DD HH:MM:SS'),
+			request_name: 'topics',
+			topics
+		}
+	};
+}
+
+async function getCommunityPeople(community: HydratedCommunityDocument, hours = 24): Promise<any> {
+	const now = new Date();
+	const last24Hours = new Date(now.getTime() - hours * 60 * 60 * 1000);
+	const people = await Post.aggregate([
+		{
+			$match: {
+				title_id: {
+					$in: community.title_id
+				},
+				created_at: {
+					$gte: last24Hours
+				},
+				message_to_pid: null,
+				parent: null,
+				removed: false
+			}
+		},
+		{
+			$group: {
+				_id: '$pid',
+				post: {
+					$first: '$$ROOT'
+				}
+			}
+		},
+		{
+			$limit: 70 // * Arbitrary
+		}
+	]);
+
+	// TODO - Remove this check once out of beta and have more users
+	// * We only do this because Juxtaposition is not super active
+	// * due to it being in beta. If we don't expand the search
+	// * time range then WWP still ends up fairly empty
+	// *
+	// * Ensure we have at *least* 20 people. Arbitrary.
+	// * If the year is less than 2020, assume we've gone
+	// * too far back. There are no more posts, just return
+	// * what was found
+	if (people.length < 20 && last24Hours.getFullYear() >= 2020) {
+		// * Double the search range each time to get
+		// * exponentially more posts. This speeds up
+		// * the search at the cost of using older posts
+		return getCommunityPeople(community, hours * 2);
+	}
+
+	return people;
 }
 
 async function calculateMostPopularCommunities(hours: number, limit: number): Promise<HydratedCommunityDocument[]> {
-	const now: Date = new Date();
-	const last24Hours: Date = new Date(now.getTime() - hours * 60 * 60 * 1000);
+	const now = new Date();
+	const last24Hours = new Date(now.getTime() - hours * 60 * 60 * 1000);
+
 	if (!last24Hours) {
 		throw new Error('Invalid date');
 	}
+
 	const validCommunities: {
 		_id: null;
 		communities: [string];
@@ -136,10 +239,13 @@ async function calculateMostPopularCommunities(hours: number, limit: number): Pr
 			}
 		}
 	]);
+
 	const communityIDs: [string] = validCommunities[0].communities;
+
 	if (!communityIDs) {
 		throw new Error('No communities found');
 	}
+
 	const popularCommunities: {
 		_id: string;
 		count: number;

src/types/common/post.ts

@@ -0,0 +1,11 @@
+export type PostPainting = {
+	format: string;
+	content: string;
+	size: number;
+	url: string;
+};
+
+export type PostScreenshot = {
+	size: number;
+	url: string;
+};

src/types/common/wara-wara-plaza.ts

@@ -0,0 +1,70 @@
+// TODO - Maybe this can become more generalized, and not specific to WWP?
+export type WWPPost = {
+	body?: string;
+	community_id: number;
+	country_id: number;
+	created_at: string;
+	feeling_id: number;
+	id: string;
+	is_autopost: 0 | 1;
+	is_community_private_autopost: 0 | 1;
+	is_spoiler: 0 | 1;
+	is_app_jumpable: 0 | 1;
+	empathy_count: number;
+	language_id: number;
+	mii: string;
+	mii_face_url: string;
+	number: number;
+	painting?: {
+		format: string;
+		content: string;
+		size: number;
+		url: string;
+	};
+	pid: number;
+	platform_id: number;
+	region_id: number;
+	reply_count: number;
+	screen_name: string;
+	screenshot?: {
+		size: number;
+		url: string;
+	};
+	title_id: string;
+};
+
+export type WWPPerson = {
+	person: {
+		posts: {
+			post: WWPPost;
+		}[];
+	}
+};
+
+export type WWPTopic = {
+	empathy_count: number;
+	has_shop_page: 0 | 1;
+	icon: string;
+	title_ids: {
+		title_id: string;
+	}[];
+	title_id: string;
+	community_id: number;
+	is_recommended: 0 | 1;
+	name: string;
+	people: WWPPerson[];
+	position: number;
+};
+
+export type WWPData = {
+	result: {
+		has_error: 0 | 1;
+		version: 1;
+		expire: string;
+		request_name: 'topics';
+		topics: {
+			topic: WWPTopic;
+		}[];
+	}
+};
+

src/types/memoizee.d.ts

@@ -1,34 +0,0 @@
-// * Taken from https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/memoizee/index.d.ts
-// * Installing from @types/memoizee did not work, types werent being seen
-
-// Type definitions for memoizee 0.4
-// Project: https://github.com/medikoo/memoizee
-// Definitions by: Juan Picado <https://github.com/juanpicado>
-//                 Patrick Muff <https://github.com/dislick>
-// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
-// Minimum TypeScript Version: 3.1
-
-declare namespace memoizee {
-	interface Options<F extends (...args: any[]) => any> {
-		length?: number | false | undefined;
-		maxAge?: number | undefined;
-		max?: number | undefined;
-		preFetch?: number | true | undefined;
-		promise?: boolean | 'then' | 'done' | 'done:finally' | undefined;
-		dispose?(value: any): void;
-		async?: boolean | undefined;
-		primitive?: boolean | undefined;
-		normalizer?(args: Parameters<F>): string;
-		resolvers?: Array<(arg: any) => any> | undefined;
-		profileName?: string;
-	}
-
-	interface Memoized<F> {
-		delete: F;
-		clear: F & (() => void);
-	}
-}
-
-declare function memoizee<F extends (...args: any[]) => any>(f: F, options?: memoizee.Options<F>): F & memoizee.Memoized<F>;
-
-export = memoizee;

src/types/mongoose/post.ts

@@ -1,6 +1,7 @@
 import { Model, Types, HydratedDocument } from 'mongoose';
 import { HydratedCommunityDocument } from '@/types/mongoose/community';
 import { PostToJSONOptions } from '@/types/mongoose/post-to-json-options';
+import { PostPainting, PostScreenshot } from '@/types/common/post';
 
 export interface IPost {
 	id: string;
@@ -44,6 +45,12 @@ export interface IPostMethods {
 	remove(reason: string): Promise<void>;
 	unRemove(reason: string): Promise<void>;
     generatePostUID(length: number): Promise<void>;
+	cleanedBody(): string;
+	cleanedMiiData(): string;
+	cleanedPainting(): string;
+	cleanedAppData(): string;
+	formatPainting(): PostPainting | undefined;
+	formatScreenshot(): PostScreenshot | undefined;
     json(options: PostToJSONOptions, community?: HydratedCommunityDocument): Record<string, any>;
 }