mirror of
https://github.com/PretendoNetwork/miiverse-api.git
synced 2026-07-12 22:41:18 -05:00
make wwp more accurate and stable
This commit is contained in:
Jonathan Barrow
parent
a9336a0e9b
commit
7d86ea3d92
130
package-lock.json
generated
130
package-lock.json
generated
|
|
@ -16,7 +16,6 @@
|
|||
"express": "^4.17.1",
|
||||
"express-subdomain": "^1.0.5",
|
||||
"fs-extra": "^9.0.0",
|
||||
"memoizee": "^0.4.15",
|
||||
"moment": "^2.24.0",
|
||||
"mongoose": "^6.10.1",
|
||||
"morgan": "^1.10.0",
|
||||
|
|
@ -34,7 +33,6 @@
|
|||
"devDependencies": {
|
||||
"@types/express": "^4.17.17",
|
||||
"@types/fs-extra": "^11.0.1",
|
||||
"@types/memoizee": "^0.4.8",
|
||||
"@types/morgan": "^1.9.4",
|
||||
"@types/multer": "^1.4.7",
|
||||
"@types/newman": "^5.3.2",
|
||||
|
|
@ -1488,12 +1486,6 @@
|
|||
"resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
|
||||
"integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA=="
|
||||
},
|
||||
"node_modules/@types/memoizee": {
|
||||
"version": "0.4.8",
|
||||
"resolved": "https://registry.npmjs.org/@types/memoizee/-/memoizee-0.4.8.tgz",
|
||||
"integrity": "sha512-qDpXKGgwKywnQt/64fH1O0LiPA++QGIYeykEUiZ51HymKVRLnUSGcRuF60IfpPeeXiuRwiR/W4y7S5VzbrgLCA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/mime": {
|
||||
"version": "1.3.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
|
||||
|
|
@ -2635,15 +2627,6 @@
|
|||
"integrity": "sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/d": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz",
|
||||
"integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==",
|
||||
"dependencies": {
|
||||
"es5-ext": "^0.10.50",
|
||||
"type": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/dashdash": {
|
||||
"version": "1.14.1",
|
||||
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
|
||||
|
|
@ -2776,50 +2759,6 @@
|
|||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/es5-ext": {
|
||||
"version": "0.10.62",
|
||||
"resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.62.tgz",
|
||||
"integrity": "sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==",
|
||||
"hasInstallScript": true,
|
||||
"dependencies": {
|
||||
"es6-iterator": "^2.0.3",
|
||||
"es6-symbol": "^3.1.3",
|
||||
"next-tick": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/es6-iterator": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz",
|
||||
"integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==",
|
||||
"dependencies": {
|
||||
"d": "1",
|
||||
"es5-ext": "^0.10.35",
|
||||
"es6-symbol": "^3.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/es6-symbol": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz",
|
||||
"integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==",
|
||||
"dependencies": {
|
||||
"d": "^1.0.1",
|
||||
"ext": "^1.1.2"
|
||||
}
|
||||
},
|
||||
"node_modules/es6-weak-map": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.3.tgz",
|
||||
"integrity": "sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==",
|
||||
"dependencies": {
|
||||
"d": "1",
|
||||
"es5-ext": "^0.10.46",
|
||||
"es6-iterator": "^2.0.3",
|
||||
"es6-symbol": "^3.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/escalade": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
|
||||
|
|
@ -3050,15 +2989,6 @@
|
|||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/event-emitter": {
|
||||
"version": "0.3.5",
|
||||
"resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz",
|
||||
"integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==",
|
||||
"dependencies": {
|
||||
"d": "1",
|
||||
"es5-ext": "~0.10.14"
|
||||
}
|
||||
},
|
||||
"node_modules/eventemitter3": {
|
||||
"version": "4.0.7",
|
||||
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
|
||||
|
|
@ -3132,19 +3062,6 @@
|
|||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
|
||||
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
|
||||
},
|
||||
"node_modules/ext": {
|
||||
"version": "1.7.0",
|
||||
"resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz",
|
||||
"integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==",
|
||||
"dependencies": {
|
||||
"type": "^2.7.2"
|
||||
}
|
||||
},
|
||||
"node_modules/ext/node_modules/type": {
|
||||
"version": "2.7.2",
|
||||
"resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz",
|
||||
"integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw=="
|
||||
},
|
||||
"node_modules/extend": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
|
||||
|
|
@ -3910,11 +3827,6 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/is-promise": {
|
||||
"version": "2.2.2",
|
||||
"resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz",
|
||||
"integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ=="
|
||||
},
|
||||
"node_modules/is-typed-array": {
|
||||
"version": "1.1.10",
|
||||
"resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.10.tgz",
|
||||
|
|
@ -4173,14 +4085,6 @@
|
|||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/lru-queue": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz",
|
||||
"integrity": "sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ==",
|
||||
"dependencies": {
|
||||
"es5-ext": "~0.10.2"
|
||||
}
|
||||
},
|
||||
"node_modules/media-typer": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
|
||||
|
|
@ -4189,21 +4093,6 @@
|
|||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/memoizee": {
|
||||
"version": "0.4.15",
|
||||
"resolved": "https://registry.npmjs.org/memoizee/-/memoizee-0.4.15.tgz",
|
||||
"integrity": "sha512-UBWmJpLZd5STPm7PMUlOw/TSy972M+z8gcyQ5veOnSDRREz/0bmpyTfKt3/51DhEBqCZQn1udM/5flcSPYhkdQ==",
|
||||
"dependencies": {
|
||||
"d": "^1.0.1",
|
||||
"es5-ext": "^0.10.53",
|
||||
"es6-weak-map": "^2.0.3",
|
||||
"event-emitter": "^0.3.5",
|
||||
"is-promise": "^2.2.2",
|
||||
"lru-queue": "^0.1.0",
|
||||
"next-tick": "^1.1.0",
|
||||
"timers-ext": "^0.1.7"
|
||||
}
|
||||
},
|
||||
"node_modules/memory-pager": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
|
||||
|
|
@ -4629,11 +4518,6 @@
|
|||
"uuid": "dist/bin/uuid"
|
||||
}
|
||||
},
|
||||
"node_modules/next-tick": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz",
|
||||
"integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ=="
|
||||
},
|
||||
"node_modules/nice-grpc": {
|
||||
"version": "2.1.4",
|
||||
"resolved": "https://registry.npmjs.org/nice-grpc/-/nice-grpc-2.1.4.tgz",
|
||||
|
|
@ -6003,15 +5887,6 @@
|
|||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
|
||||
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
|
||||
},
|
||||
"node_modules/timers-ext": {
|
||||
"version": "0.1.7",
|
||||
"resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz",
|
||||
"integrity": "sha512-b85NUNzTSdodShTIbky6ZF02e8STtVVfD+fu4aXXShEELpozH+bCpJLYMPZbsABN2wDH7fJpqIoXxJpzbf0NqQ==",
|
||||
"dependencies": {
|
||||
"es5-ext": "~0.10.46",
|
||||
"next-tick": "1"
|
||||
}
|
||||
},
|
||||
"node_modules/to-regex-range": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
|
||||
|
|
@ -6157,11 +6032,6 @@
|
|||
"integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/type": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz",
|
||||
"integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg=="
|
||||
},
|
||||
"node_modules/type-check": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@
|
|||
"express": "^4.17.1",
|
||||
"express-subdomain": "^1.0.5",
|
||||
"fs-extra": "^9.0.0",
|
||||
"memoizee": "^0.4.15",
|
||||
"moment": "^2.24.0",
|
||||
"mongoose": "^6.10.1",
|
||||
"morgan": "^1.10.0",
|
||||
|
|
@ -39,7 +38,6 @@
|
|||
"devDependencies": {
|
||||
"@types/express": "^4.17.17",
|
||||
"@types/fs-extra": "^11.0.1",
|
||||
"@types/memoizee": "^0.4.8",
|
||||
"@types/morgan": "^1.9.4",
|
||||
"@types/multer": "^1.4.7",
|
||||
"@types/newman": "^5.3.2",
|
||||
|
|
|
|||
27
src/cache.ts
Normal file
27
src/cache.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
export default class Cache<T> {
|
||||
private data?: T;
|
||||
private expireAt: number;
|
||||
private cacheTime: number;
|
||||
|
||||
constructor(cacheTime: number) {
|
||||
this.expireAt = Date.now() + cacheTime;
|
||||
this.cacheTime = cacheTime;
|
||||
}
|
||||
|
||||
valid(): boolean {
|
||||
if (!this.data || Date.now() >= this.expireAt) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
update(data: T): void {
|
||||
this.expireAt = Date.now() + this.cacheTime;
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
get(): T | undefined {
|
||||
return this.data;
|
||||
}
|
||||
}
|
||||
|
|
@ -52,6 +52,7 @@ async function auth(request: express.Request, response: express.Response, next:
|
|||
const paramPackData: ParamPack = decodeParamPack(paramPack);
|
||||
const paramPackCheck: z.SafeParseReturnType<ParamPack, ParamPack> = ParamPackSchema.safeParse(paramPackData);
|
||||
if (!paramPackCheck.success) {
|
||||
console.log(paramPackCheck.error);
|
||||
return badAuth(response, 18, 'BAD_PARAM');
|
||||
}
|
||||
|
||||
|
|
@ -61,6 +62,7 @@ async function auth(request: express.Request, response: express.Response, next:
|
|||
user = await getUserAccountData(pid);
|
||||
} catch (error) {
|
||||
// TODO - Log this error
|
||||
console.log(error);
|
||||
return badAuth(response, 18, 'BAD_PARAM');
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { Schema, model } from 'mongoose';
|
|||
import { HydratedPostDocument, IPost, IPostMethods, PostModel } from '@/types/mongoose/post';
|
||||
import { HydratedCommunityDocument } from '@/types/mongoose/community';
|
||||
import { PostToJSONOptions } from '@/types/mongoose/post-to-json-options';
|
||||
import { PostPainting, PostScreenshot } from '@/types/common/post';
|
||||
|
||||
const PostSchema = new Schema<IPost, PostModel, IPostMethods>({
|
||||
id: String,
|
||||
|
|
@ -132,9 +133,45 @@ PostSchema.method('generatePostUID', async function generatePostUID(length: numb
|
|||
}
|
||||
});
|
||||
|
||||
PostSchema.method('cleanedBody', function cleanedBody(): string {
|
||||
return this.body ? this.body.replace(/[^A-Za-z\d\s-_!@#$%^&*(){}+=,.<>/?;:'"[\]]/g, '').replace(/[\n\r]+/gm, '') : '';
|
||||
});
|
||||
|
||||
PostSchema.method('cleanedMiiData', function cleanedMiiData(): string {
|
||||
return this.mii.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim();
|
||||
});
|
||||
|
||||
PostSchema.method('cleanedPainting', function cleanedPainting(): string {
|
||||
return this.painting.replace(/[\n\r]+/gm, '').trim();
|
||||
});
|
||||
|
||||
PostSchema.method('cleanedAppData', function cleanedAppData(): string {
|
||||
return this.app_data.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim();
|
||||
});
|
||||
|
||||
PostSchema.method('formatPainting', function formatPainting(): PostPainting | undefined {
|
||||
if (this.painting) {
|
||||
return {
|
||||
format: 'tga',
|
||||
content: this.cleanedPainting(),
|
||||
size: this.painting.length,
|
||||
url: `https://pretendo-cdn.b-cdn.net/paintings/${this.pid}/${this.id}.png`
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
PostSchema.method('formatScreenshot', function formatScreenshot(): PostScreenshot | undefined {
|
||||
if (this.screenshot && this.screenshot_length) {
|
||||
return {
|
||||
size: this.screenshot_length,
|
||||
url: `https://pretendo-cdn.b-cdn.net/screenshots/${this.pid}/${this.id}.jpg`
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
PostSchema.method('json', function json(options: PostToJSONOptions, community?: HydratedCommunityDocument): Record<string, any> {
|
||||
const json: Record<string, any> = {
|
||||
body: this.body ? this.body.replace(/[^A-Za-z\d\s-_!@#$%^&*(){}+=,.<>/?;:'"[\]]/g, '').replace(/[\n\r]+/gm, '') : '',
|
||||
body: this.cleanedBody(),
|
||||
country_id: this.country_id ? this.country_id : 254,
|
||||
created_at: moment(this.created_at).format('YYYY-MM-DD HH:MM:SS'),
|
||||
feeling_id: this.feeling_id,
|
||||
|
|
@ -155,7 +192,7 @@ PostSchema.method('json', function json(options: PostToJSONOptions, community?:
|
|||
};
|
||||
|
||||
if (this.app_data && options.app_data) {
|
||||
json.app_data = this.app_data.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim();
|
||||
json.app_data = this.cleanedAppData();
|
||||
}
|
||||
|
||||
if (options.topics && community) {
|
||||
|
|
@ -165,24 +202,16 @@ PostSchema.method('json', function json(options: PostToJSONOptions, community?:
|
|||
}
|
||||
|
||||
if (options.with_mii) {
|
||||
json.mii = this.mii.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim();
|
||||
json.mii = this.cleanedMiiData();
|
||||
json.mii_face_url = this.mii_face_url;
|
||||
}
|
||||
|
||||
if (this.painting) {
|
||||
json.painting = {
|
||||
format: 'tga',
|
||||
content: this.painting.replace(/[\n\r]+/gm, '').trim(),
|
||||
size: this.painting.length,
|
||||
url: `https://pretendo-cdn.b-cdn.net/paintings/${this.pid}/${this.id}.png`
|
||||
};
|
||||
json.painting = this.formatPainting();
|
||||
}
|
||||
|
||||
if (this.screenshot && this.screenshot_length) {
|
||||
json.screenshot = {
|
||||
size: this.screenshot_length,
|
||||
url: `https://pretendo-cdn.b-cdn.net/screenshots/${this.pid}/${this.id}.jpg`
|
||||
};
|
||||
json.screenshot = this.formatScreenshot();
|
||||
}
|
||||
|
||||
if (this.topic_tag && options.topic_tag) {
|
||||
|
|
|
|||
|
|
@ -1,23 +1,19 @@
|
|||
import express from 'express';
|
||||
import memoize from 'memoizee';
|
||||
import moment from 'moment';
|
||||
import xmlbuilder from 'xmlbuilder';
|
||||
import { GetUserDataResponse } from 'pretendo-grpc-ts/dist/account/get_user_data_rpc';
|
||||
import { getUserAccountData } from '@/util';
|
||||
import { getEndpoint, getPostsBytitleID } from '@/database';
|
||||
import Cache from '@/cache';
|
||||
import { getEndpoint } from '@/database';
|
||||
import { Post } from '@/models/post';
|
||||
import { Community } from '@/models/community';
|
||||
import { HydratedEndpointDocument } from '@/types/mongoose/endpoint';
|
||||
import { HydratedCommunityDocument } from '@/types/mongoose/community';
|
||||
import { HydratedPostDocument } from '@/types/mongoose/post';
|
||||
import { WWPData, WWPPost, WWPTopic } from '@/types/common/wara-wara-plaza';
|
||||
|
||||
const router: express.Router = express.Router();
|
||||
|
||||
// TODO - Need to add types to memoize in @/types/memoize.d.ts
|
||||
const memoizedGenerateTopicsXML = memoize(generateTopicsXML, {
|
||||
async: true,
|
||||
maxAge: 1000 * 60 * 60 // * cache for 1 hour
|
||||
});
|
||||
const router = express.Router();
|
||||
const ONE_HOUR = 60 * 60 * 1000;
|
||||
const WARA_WARA_PLAZA_CACHE = new Cache<WWPData>(ONE_HOUR);
|
||||
|
||||
/* GET post titles. */
|
||||
router.get('/', async function (request: express.Request, response: express.Response): Promise<void> {
|
||||
|
|
@ -46,77 +42,184 @@ router.get('/', async function (request: express.Request, response: express.Resp
|
|||
return;
|
||||
}
|
||||
|
||||
const communities: HydratedCommunityDocument[] = await calculateMostPopularCommunities(24, 10);
|
||||
if (!WARA_WARA_PLAZA_CACHE.valid()) {
|
||||
const communities: HydratedCommunityDocument[] = await calculateMostPopularCommunities(24, 10);
|
||||
|
||||
if (communities.length < 10) {
|
||||
response.sendStatus(404);
|
||||
return;
|
||||
if (communities.length < 10) {
|
||||
response.sendStatus(404);
|
||||
return;
|
||||
}
|
||||
|
||||
WARA_WARA_PLAZA_CACHE.update(await generateTopicsData(communities));
|
||||
}
|
||||
|
||||
response.send(await memoizedGenerateTopicsXML(communities));
|
||||
const data = WARA_WARA_PLAZA_CACHE.get() || {};
|
||||
|
||||
response.send(xmlbuilder.create(data, { separateArrayItems: true }).end({ pretty: true, allowEmpty: true }));
|
||||
});
|
||||
|
||||
async function generateTopicsXML(communities: HydratedCommunityDocument[]): Promise<string> {
|
||||
const json: Record<string, any> = {
|
||||
result: {
|
||||
has_error: 0,
|
||||
version: 1,
|
||||
expire: moment().add(2, 'days').format('YYYY-MM-DD HH:MM:SS'),
|
||||
request_name: 'topics',
|
||||
topics: []
|
||||
}
|
||||
};
|
||||
async function generateTopicsData(communities: HydratedCommunityDocument[]): Promise<WWPData> {
|
||||
const topics: {
|
||||
topic: WWPTopic;
|
||||
}[] = [];
|
||||
|
||||
for (const community of communities) {
|
||||
const topic: Record<string, any> = {
|
||||
topic: {
|
||||
empathy_count: community.empathy_count,
|
||||
has_shop_page: community.has_shop_page,
|
||||
icon: community.icon,
|
||||
title_ids: [],
|
||||
title_id: community.title_id[0],
|
||||
community_id: community.community_id,
|
||||
is_recommended: community.is_recommended,
|
||||
name: community.name,
|
||||
people: []
|
||||
for (let i = 0; i < communities.length; i++) {
|
||||
const community = communities[i];
|
||||
|
||||
const empathies = await Post.aggregate([
|
||||
{
|
||||
$match: {
|
||||
community_id: community.olive_community_id
|
||||
}
|
||||
},
|
||||
{
|
||||
$group : {
|
||||
_id : null,
|
||||
total : {
|
||||
$sum : '$empathy_count'
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
$limit: 1
|
||||
}
|
||||
]);
|
||||
|
||||
const topic: WWPTopic = {
|
||||
empathy_count: empathies[0]?.total || 0,
|
||||
has_shop_page: community.has_shop_page ? 1 : 0,
|
||||
icon: community.icon,
|
||||
title_ids: [],
|
||||
title_id: community.title_id[0],
|
||||
community_id: 0xFFFFFFFF, // * This is how it was in the real WWP. Unsure why, but it works
|
||||
is_recommended: community.is_recommended ? 1 : 0,
|
||||
name: community.name,
|
||||
people: [],
|
||||
position: i+1
|
||||
};
|
||||
|
||||
community.title_id.forEach(function (title_id: string) {
|
||||
if (title_id !== '') {
|
||||
topic.topic.title_ids.push({ title_id });
|
||||
community.title_id.forEach(title_id => {
|
||||
// * Just in case
|
||||
if (title_id) {
|
||||
topic.title_ids.push({ title_id });
|
||||
}
|
||||
});
|
||||
|
||||
const posts: HydratedPostDocument[] = await getPostsBytitleID(community.title_id, 30);
|
||||
const people = await getCommunityPeople(community);
|
||||
|
||||
for (const post of posts) {
|
||||
topic.topic.people.push({
|
||||
for (const person of people) {
|
||||
const hydratedPost = Post.hydrate(person.post);
|
||||
|
||||
const post: WWPPost = {
|
||||
body: hydratedPost.cleanedBody(),
|
||||
community_id: 0xFFFFFFFF, // * This is how it was in the real WWP. Unsure why, but it works
|
||||
country_id: hydratedPost.country_id,
|
||||
created_at: moment(hydratedPost.created_at).format('YYYY-MM-DD HH:MM:SS'),
|
||||
feeling_id: hydratedPost.feeling_id,
|
||||
id: hydratedPost.id,
|
||||
is_autopost: hydratedPost.is_autopost ? 1 : 0,
|
||||
is_community_private_autopost: hydratedPost.is_community_private_autopost ? 1 : 0,
|
||||
is_spoiler: hydratedPost.is_spoiler ? 1 : 0,
|
||||
is_app_jumpable: hydratedPost.is_app_jumpable ? 1 : 0,
|
||||
empathy_count: hydratedPost.empathy_count || 0,
|
||||
language_id: hydratedPost.language_id,
|
||||
mii: hydratedPost.cleanedMiiData(),
|
||||
mii_face_url: hydratedPost.mii_face_url,
|
||||
number: 0,
|
||||
painting: hydratedPost.formatPainting(),
|
||||
pid: hydratedPost.pid,
|
||||
platform_id: hydratedPost.platform_id,
|
||||
region_id: hydratedPost.region_id,
|
||||
reply_count: hydratedPost.reply_count || 0,
|
||||
screen_name: hydratedPost.screen_name,
|
||||
screenshot: hydratedPost.formatScreenshot(),
|
||||
title_id: hydratedPost.title_id,
|
||||
};
|
||||
|
||||
topic.people.push({
|
||||
person: {
|
||||
posts: [
|
||||
{
|
||||
post: post.json({
|
||||
with_mii: true,
|
||||
topics: true
|
||||
})
|
||||
post
|
||||
}
|
||||
]
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
json.result.topics.push(topic);
|
||||
topics.push({
|
||||
topic: topic
|
||||
});
|
||||
}
|
||||
|
||||
return xmlbuilder.create(json, { separateArrayItems: true }).end({ pretty: true, allowEmpty: true });
|
||||
return {
|
||||
result: {
|
||||
has_error: 0,
|
||||
version: 1,
|
||||
expire: moment().add(2, 'days').format('YYYY-MM-DD HH:MM:SS'),
|
||||
request_name: 'topics',
|
||||
topics
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function getCommunityPeople(community: HydratedCommunityDocument, hours = 24): Promise<any> {
|
||||
const now = new Date();
|
||||
const last24Hours = new Date(now.getTime() - hours * 60 * 60 * 1000);
|
||||
const people = await Post.aggregate([
|
||||
{
|
||||
$match: {
|
||||
title_id: {
|
||||
$in: community.title_id
|
||||
},
|
||||
created_at: {
|
||||
$gte: last24Hours
|
||||
},
|
||||
message_to_pid: null,
|
||||
parent: null,
|
||||
removed: false
|
||||
}
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
_id: '$pid',
|
||||
post: {
|
||||
$first: '$$ROOT'
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
$limit: 70 // * Arbitrary
|
||||
}
|
||||
]);
|
||||
|
||||
// TODO - Remove this check once out of beta and have more users
|
||||
// * We only do this because Juxtaposition is not super active
|
||||
// * due to it being in beta. If we don't expand the search
|
||||
// * time range then WWP still ends up fairly empty
|
||||
// *
|
||||
// * Ensure we have at *least* 20 people. Arbitrary.
|
||||
// * If the year is less than 2020, assume we've gone
|
||||
// * too far back. There are no more posts, just return
|
||||
// * what was found
|
||||
if (people.length < 20 && last24Hours.getFullYear() >= 2020) {
|
||||
// * Double the search range each time to get
|
||||
// * exponentially more posts. This speeds up
|
||||
// * the search at the cost of using older posts
|
||||
return getCommunityPeople(community, hours * 2);
|
||||
}
|
||||
|
||||
return people;
|
||||
}
|
||||
|
||||
async function calculateMostPopularCommunities(hours: number, limit: number): Promise<HydratedCommunityDocument[]> {
|
||||
const now: Date = new Date();
|
||||
const last24Hours: Date = new Date(now.getTime() - hours * 60 * 60 * 1000);
|
||||
const now = new Date();
|
||||
const last24Hours = new Date(now.getTime() - hours * 60 * 60 * 1000);
|
||||
|
||||
if (!last24Hours) {
|
||||
throw new Error('Invalid date');
|
||||
}
|
||||
|
||||
const validCommunities: {
|
||||
_id: null;
|
||||
communities: [string];
|
||||
|
|
@ -136,10 +239,13 @@ async function calculateMostPopularCommunities(hours: number, limit: number): Pr
|
|||
}
|
||||
}
|
||||
]);
|
||||
|
||||
const communityIDs: [string] = validCommunities[0].communities;
|
||||
|
||||
if (!communityIDs) {
|
||||
throw new Error('No communities found');
|
||||
}
|
||||
|
||||
const popularCommunities: {
|
||||
_id: string;
|
||||
count: number;
|
||||
|
|
|
|||
11
src/types/common/post.ts
Normal file
11
src/types/common/post.ts
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
export type PostPainting = {
|
||||
format: string;
|
||||
content: string;
|
||||
size: number;
|
||||
url: string;
|
||||
};
|
||||
|
||||
export type PostScreenshot = {
|
||||
size: number;
|
||||
url: string;
|
||||
};
|
||||
70
src/types/common/wara-wara-plaza.ts
Normal file
70
src/types/common/wara-wara-plaza.ts
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
// TODO - Maybe this can become more generalized, and not specific to WWP?
|
||||
export type WWPPost = {
|
||||
body?: string;
|
||||
community_id: number;
|
||||
country_id: number;
|
||||
created_at: string;
|
||||
feeling_id: number;
|
||||
id: string;
|
||||
is_autopost: 0 | 1;
|
||||
is_community_private_autopost: 0 | 1;
|
||||
is_spoiler: 0 | 1;
|
||||
is_app_jumpable: 0 | 1;
|
||||
empathy_count: number;
|
||||
language_id: number;
|
||||
mii: string;
|
||||
mii_face_url: string;
|
||||
number: number;
|
||||
painting?: {
|
||||
format: string;
|
||||
content: string;
|
||||
size: number;
|
||||
url: string;
|
||||
};
|
||||
pid: number;
|
||||
platform_id: number;
|
||||
region_id: number;
|
||||
reply_count: number;
|
||||
screen_name: string;
|
||||
screenshot?: {
|
||||
size: number;
|
||||
url: string;
|
||||
};
|
||||
title_id: string;
|
||||
};
|
||||
|
||||
export type WWPPerson = {
|
||||
person: {
|
||||
posts: {
|
||||
post: WWPPost;
|
||||
}[];
|
||||
}
|
||||
};
|
||||
|
||||
export type WWPTopic = {
|
||||
empathy_count: number;
|
||||
has_shop_page: 0 | 1;
|
||||
icon: string;
|
||||
title_ids: {
|
||||
title_id: string;
|
||||
}[];
|
||||
title_id: string;
|
||||
community_id: number;
|
||||
is_recommended: 0 | 1;
|
||||
name: string;
|
||||
people: WWPPerson[];
|
||||
position: number;
|
||||
};
|
||||
|
||||
export type WWPData = {
|
||||
result: {
|
||||
has_error: 0 | 1;
|
||||
version: 1;
|
||||
expire: string;
|
||||
request_name: 'topics';
|
||||
topics: {
|
||||
topic: WWPTopic;
|
||||
}[];
|
||||
}
|
||||
};
|
||||
|
||||
34
src/types/memoizee.d.ts
vendored
34
src/types/memoizee.d.ts
vendored
|
|
@ -1,34 +0,0 @@
|
|||
// * Taken from https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/memoizee/index.d.ts
|
||||
// * Installing from @types/memoizee did not work, types werent being seen
|
||||
|
||||
// Type definitions for memoizee 0.4
|
||||
// Project: https://github.com/medikoo/memoizee
|
||||
// Definitions by: Juan Picado <https://github.com/juanpicado>
|
||||
// Patrick Muff <https://github.com/dislick>
|
||||
// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
|
||||
// Minimum TypeScript Version: 3.1
|
||||
|
||||
declare namespace memoizee {
|
||||
interface Options<F extends (...args: any[]) => any> {
|
||||
length?: number | false | undefined;
|
||||
maxAge?: number | undefined;
|
||||
max?: number | undefined;
|
||||
preFetch?: number | true | undefined;
|
||||
promise?: boolean | 'then' | 'done' | 'done:finally' | undefined;
|
||||
dispose?(value: any): void;
|
||||
async?: boolean | undefined;
|
||||
primitive?: boolean | undefined;
|
||||
normalizer?(args: Parameters<F>): string;
|
||||
resolvers?: Array<(arg: any) => any> | undefined;
|
||||
profileName?: string;
|
||||
}
|
||||
|
||||
interface Memoized<F> {
|
||||
delete: F;
|
||||
clear: F & (() => void);
|
||||
}
|
||||
}
|
||||
|
||||
declare function memoizee<F extends (...args: any[]) => any>(f: F, options?: memoizee.Options<F>): F & memoizee.Memoized<F>;
|
||||
|
||||
export = memoizee;
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import { Model, Types, HydratedDocument } from 'mongoose';
|
||||
import { HydratedCommunityDocument } from '@/types/mongoose/community';
|
||||
import { PostToJSONOptions } from '@/types/mongoose/post-to-json-options';
|
||||
import { PostPainting, PostScreenshot } from '@/types/common/post';
|
||||
|
||||
export interface IPost {
|
||||
id: string;
|
||||
|
|
@ -44,6 +45,12 @@ export interface IPostMethods {
|
|||
remove(reason: string): Promise<void>;
|
||||
unRemove(reason: string): Promise<void>;
|
||||
generatePostUID(length: number): Promise<void>;
|
||||
cleanedBody(): string;
|
||||
cleanedMiiData(): string;
|
||||
cleanedPainting(): string;
|
||||
cleanedAppData(): string;
|
||||
formatPainting(): PostPainting | undefined;
|
||||
formatScreenshot(): PostScreenshot | undefined;
|
||||
json(options: PostToJSONOptions, community?: HydratedCommunityDocument): Record<string, any>;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user