Leahnaya/miiverse-api
1
0
Fork 0
mirror of https://github.com/PretendoNetwork/miiverse-api.git synced 2026-03-21 17:24:38 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Removed account server DB, now uses gRPC

Browse Source
This commit is contained in:
Jonathan Barrow 2023-04-28 14:13:03 -04:00
parent 1b76eb967b
commit 436e5f3bd9
No known key found for this signature in database
GPG Key ID: E86E9FE9049C741F
14 changed files with 1487 additions and 954 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2089
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

40
src/accountdb.ts
View File

@ -1,40 +0,0 @@
import mongoose from 'mongoose';
import { LOG_INFO, LOG_ERROR } from '@/logger';
import { config } from '@/config-manager';
const { account_db: mongooseConfig } = config;
export let pnidConnection: mongoose.Connection;
export function connect(): void {
if (!pnidConnection) {
pnidConnection = makeNewConnection(mongooseConfig.connection_string);
}
}
export function verifyConnected(): void {
if (!pnidConnection) {
throw new Error('Cannot make database requests without being connected');
}
}
export function makeNewConnection(uri: string): mongoose.Connection {
pnidConnection = mongoose.createConnection(uri, mongooseConfig.options);
pnidConnection.on('error', error => {
LOG_ERROR(`MongoDB connection ${JSON.stringify(error)}`);
pnidConnection.close().catch(error => LOG_ERROR(JSON.stringify(error)));
});
pnidConnection.on('connected', () => {
LOG_INFO(`MongoDB connected ${uri}`);
});
pnidConnection.on('disconnected', () => {
LOG_INFO('MongoDB disconnected');
});
return pnidConnection;
}
pnidConnection = makeNewConnection(mongooseConfig.connection_string);

37
src/config-manager.ts
View File

@ -16,14 +16,6 @@ if (process.env.PN_MIIVERSE_API_CONFIG_MONGOOSE_CONNECT_OPTIONS_PATH) {
LOG_WARN('No Mongoose connection options found for main connection. To add connection options, set PN_MIIVERSE_API_CONFIG_MONGOOSE_CONNECT_OPTIONS_PATH to the path of your options JSON file');
}
let mongooseConnectOptionsAccount: mongoose.ConnectOptions = {};
if (process.env.PN_MIIVERSE_API_CONFIG_MONGOOSE_ACCOUNT_SERVER_CONNECT_OPTIONS_PATH) {
mongooseConnectOptionsAccount = fs.readJSONSync(process.env.PN_MIIVERSE_API_CONFIG_MONGOOSE_ACCOUNT_SERVER_CONNECT_OPTIONS_PATH);
} else {
LOG_WARN('No Mongoose connection options found for main connection. To add connection options, set PN_MIIVERSE_API_CONFIG_MONGOOSE_ACCOUNT_SERVER_CONNECT_OPTIONS_PATH to the path of your options JSON file');
}
export const config: Config = {
http: {
port: Number(process.env.PN_MIIVERSE_API_CONFIG_HTTP_PORT || '')
@ -34,10 +26,6 @@ export const config: Config = {
connection_string: process.env.PN_MIIVERSE_API_CONFIG_MONGO_CONNECTION_STRING || '',
options: mongooseConnectOptionsMain
},
account_db: {
connection_string: process.env.PN_MIIVERSE_API_CONFIG_MONGO_ACCOUNT_SERVER_CONNECTION_STRING || '',
options: mongooseConnectOptionsAccount
},
s3: {
endpoint: process.env.PN_MIIVERSE_API_CONFIG_S3_ENDPOINT || '',
key: process.env.PN_MIIVERSE_API_CONFIG_S3_ACCESS_KEY || '',
@ -48,6 +36,11 @@ export const config: Config = {
ip: process.env.PN_MIIVERSE_API_CONFIG_GRPC_FRIENDS_IP || '',
port: Number(process.env.PN_MIIVERSE_API_CONFIG_GRPC_FRIENDS_PORT || ''),
api_key: process.env.PN_MIIVERSE_API_CONFIG_GRPC_FRIENDS_API_KEY || ''
},
account: {
ip: process.env.PN_MIIVERSE_API_CONFIG_GRPC_ACCOUNT_IP || '',
port: Number(process.env.PN_MIIVERSE_API_CONFIG_GRPC_ACCOUNT_PORT || ''),
api_key: process.env.PN_MIIVERSE_API_CONFIG_GRPC_ACCOUNT_API_KEY || ''
}
}
};
@ -74,11 +67,6 @@ if (!config.mongoose.connection_string) {
process.exit(0);
}
if (!config.account_db.connection_string) {
LOG_ERROR('Failed to find MongoDB Account Server connection string. Set the PN_MIIVERSE_API_CONFIG_MONGO_ACCOUNT_SERVER_CONNECTION_STRING environment variable');
process.exit(0);
}
if (!config.s3.endpoint) {
LOG_ERROR('Failed to find s3 endpoint. Set the PN_MIIVERSE_API_CONFIG_S3_ENDPOINT environment variable');
process.exit(0);
@ -107,4 +95,19 @@ if (!config.grpc.friends.port) {
if (!config.grpc.friends.api_key) {
LOG_ERROR('Failed to find NEX Friends gRPC API key. Set the PN_MIIVERSE_API_CONFIG_GRPC_FRIENDS_API_KEY environment variable');
process.exit(0);
}
if (!config.grpc.account.ip) {
LOG_ERROR('Failed to find account server gRPC ip. Set the PN_MIIVERSE_API_CONFIG_GRPC_ACCOUNT_IP environment variable');
process.exit(0);
}
if (!config.grpc.account.port) {
LOG_ERROR('Failed to find account server gRPC port. Set the PN_MIIVERSE_API_CONFIG_GRPC_ACCOUNT_PORT environment variable');
process.exit(0);
}
if (!config.grpc.account.api_key) {
LOG_ERROR('Failed to find account server gRPC API key. Set the PN_MIIVERSE_API_CONFIG_GRPC_ACCOUNT_API_KEY environment variable');
process.exit(0);
}

11
src/database.ts
View File

@ -1,16 +1,13 @@
import mongoose from 'mongoose';
import { verifyConnected as accountDBVerifyConnected } from '@/accountdb';
import { LOG_INFO } from '@/logger';
import { Community } from '@/models/community';
import { Content } from '@/models/content';
import { Conversation } from '@/models/conversation';
import { Endpoint } from '@/models/endpoint';
import { PNID } from '@/models/pnid';
import { Post } from '@/models/post';
import { Settings } from '@/models/settings';
import { config } from '@/config-manager';
import { HydratedCommunityDocument } from '@/types/mongoose/community';
import { HydratedPNIDDocument } from '@/types/mongoose/pnid';
import { HydratedPostDocument, IPost } from '@/types/mongoose/post';
import { HydratedEndpointDocument } from '@/types/mongoose/endpoint';
import { HydratedSettingsDocument } from '@/types/mongoose/settings';
@ -179,12 +176,4 @@ export async function getFriendMessages(pid: string, search_key: string[], limit
parent: null,
removed: false
}).sort({ created_at: 1 }).limit(limit);
}
export async function getPNID(pid: number): Promise<HydratedPNIDDocument | null> {
accountDBVerifyConnected();
return PNID.findOne({
pid: pid
});
}

18
src/middleware/auth.ts
View File

@ -1,11 +1,11 @@
import express from 'express';
import xmlbuilder from 'xmlbuilder';
import { z } from 'zod';
import { getPNID, getEndpoint } from '@/database';
import { getValueFromHeaders, decodeParamPack, getPIDFromServiceToken } from '@/util';
import { GetUserDataResponse } from 'pretendo-grpc-ts/dist/account/get_user_data_rpc';
import { getEndpoint } from '@/database';
import { getUserAccountData, getValueFromHeaders, decodeParamPack, getPIDFromServiceToken } from '@/util';
import { ParamPack } from '@/types/common/param-pack';
import { HydratedEndpointDocument } from '@/types/mongoose/endpoint';
import { HydratedPNIDDocument } from '@/types/mongoose/pnid';
const ParamPackSchema = z.object({
title_id: z.string(),
@ -55,10 +55,18 @@ async function auth(request: express.Request, response: express.Response, next:
return badAuth(response, 18, 'BAD_PARAM');
}
const user: HydratedPNIDDocument | null = await getPNID(pid);
let user: GetUserDataResponse;
try {
user = await getUserAccountData(pid);
} catch (error) {
// TODO - Log this error
return badAuth(response, 18, 'BAD_PARAM');
}
let discovery: HydratedEndpointDocument | null;
if (user) {
discovery = await getEndpoint(user.server_access_level);
discovery = await getEndpoint(user.serverAccessLevel);
} else {
discovery = await getEndpoint('prod');
}

38
src/models/pnid.ts
View File

@ -1,38 +0,0 @@
import mongoose from 'mongoose';
import { pnidConnection } from '@/accountdb';
import { IPNID, IPNIDMethods, PNIDModel } from '@/types/mongoose/pnid';
const PNIDSchema = new mongoose.Schema<IPNID, PNIDModel, IPNIDMethods>({
access_level: {
type: Number,
default: 0 // -1: banned, 0: standard, 1: tester, 2: mod, 3: dev
},
server_access_level: {
type: String,
default: 'prod' // prod, test, dev
},
pid: {
type: Number,
unique: true
},
username: String,
birthdate: String,
country: String,
mii: {
name: String,
data: String,
},
connections: {
stripe: {
customer_id: String,
subscription_id: String,
price_id: String,
tier_level: Number,
tier_name: String,
latest_webhook_timestamp: Number
}
}
});
export const PNID: PNIDModel = pnidConnection.model<IPNID, PNIDModel>('PNID', PNIDSchema);

1
src/server.ts
View File

@ -72,7 +72,6 @@ async function main(): Promise<void> {
LOG_INFO('Starting server');
await connectDatabase();
// TODO - Connect to account DB here too?
app.listen(port, () => {
LOG_SUCCESS(`Server started on port ${port}`);

37
src/services/api/routes/friend_messages.ts
View File

@ -5,14 +5,14 @@ import moment from 'moment';
import xmlbuilder from 'xmlbuilder';
import { z } from 'zod';
import { ParsedQs } from 'qs';
import { getUserFriendPIDs, processPainting, uploadCDNAsset, getValueFromQueryString } from '@/util';
import { getPNID, getConversationByUsers, getUserSettings, getFriendMessages } from '@/database';
import { GetUserDataResponse } from 'pretendo-grpc-ts/dist/account/get_user_data_rpc';
import { getUserFriendPIDs, getUserAccountData, processPainting, uploadCDNAsset, getValueFromQueryString } from '@/util';
import { getConversationByUsers, getUserSettings, getFriendMessages } from '@/database';
import { LOG_WARN } from '@/logger';
import { Post } from '@/models/post';
import { Conversation } from '@/models/conversation';
import { SendMessageBody } from '@/types/common/send-message-body';
import { FormattedMessage } from '@/types/common/formatted-message';
import { HydratedPNIDDocument } from '@/types/mongoose/pnid';
import { HydratedConversationDocument } from '@/types/mongoose/conversation';
import { HydratedSettingsDocument } from '@/types/mongoose/settings';
import { HydratedPostDocument } from '@/types/mongoose/post';
@ -50,10 +50,29 @@ router.post('/', upload.none(), async function (request: express.Request, respon
return;
}
const sender: HydratedPNIDDocument | null = await getPNID(request.pid);
const recipient: HydratedPNIDDocument | null = await getPNID(recipientPID);
let sender: GetUserDataResponse;
if (!sender || !recipient) {
try {
sender = await getUserAccountData(request.pid);
} catch (error) {
// TODO - Log this error
response.status(422);
return;
}
if (!sender.mii) {
// * This should never happen, but TypeScript complains so check anyway
// TODO - Better errors
response.status(422);
return;
}
let recipient: GetUserDataResponse;
try {
recipient = await getUserAccountData(request.pid);
} catch (error) {
// TODO - Log this error
response.status(422);
return;
}
@ -74,12 +93,12 @@ router.post('/', upload.none(), async function (request: express.Request, respon
users: [
{
pid: sender.pid,
official: (sender.access_level === 2 || sender.access_level === 3),
official: (sender.accessLevel === 2 || sender.accessLevel === 3),
read: true
},
{
pid: recipient.pid,
official: (recipient.access_level === 2 || recipient.access_level === 3),
official: (recipient.accessLevel === 2 || recipient.accessLevel === 3),
read: false
},
]
@ -154,7 +173,7 @@ router.post('/', upload.none(), async function (request: express.Request, respon
pid: request.pid,
platform_id: request.paramPack.platform_id,
region_id: request.paramPack.region_id,
verified: (sender.access_level === 2 || sender.access_level === 3),
verified: (sender.accessLevel === 2 || sender.accessLevel === 3),
message_to_pid: request.body.message_to_pid,
parent: null,
removed: false

55
src/services/api/routes/posts.ts
View File

@ -2,12 +2,12 @@ import express from 'express';
import multer from 'multer';
import xmlbuilder from 'xmlbuilder';
import { z } from 'zod';
import { processPainting, uploadCDNAsset, getValueFromQueryString } from '@/util';
import { GetUserDataResponse } from 'pretendo-grpc-ts/dist/account/get_user_data_rpc';
import { getUserAccountData, processPainting, uploadCDNAsset, getValueFromQueryString } from '@/util';
import {
getPostByID,
getUserContent,
getPostReplies,
getPNID,
getUserSettings,
getCommunityByID,
getCommunityByTitleID,
@ -18,21 +18,20 @@ import { Post } from '@/models/post';
import { Community } from '@/models/community';
import { HydratedPostDocument, IPost } from '@/types/mongoose/post';
import { HydratedContentDocument } from '@/types/mongoose/content';
import { HydratedPNIDDocument } from '@/types/mongoose/pnid';
import { HydratedSettingsDocument } from '@/types/mongoose/settings';
const newPostSchema = z.object({
community_id: z.string(),
community_id: z.string().optional(),
app_data: z.string().optional(),
painting: z.string().optional(),
screenshot: z.string().optional(),
body: z.string(),
feeling_id: z.string(),
search_key: z.string().array(),
topic_tag: z.string(),
search_key: z.string().array().optional(),
topic_tag: z.string().optional(),
is_autopost: z.string(),
spoiler: z.string().optional(),
is_app_jumpable: z.string(),
is_spoiler: z.string().optional(),
is_app_jumpable: z.string().optional(),
language_id: z.string()
});
@ -206,26 +205,42 @@ router.get('/', async function (request: express.Request, response: express.Resp
async function newPost(request: express.Request, response: express.Response): Promise<void> {
response.type('application/xml');
const PNID: HydratedPNIDDocument | null = await getPNID(request.pid);
const userSettings: HydratedSettingsDocument | null = await getUserSettings(request.pid);
const bodyCheck = newPostSchema.safeParse(request.body);
let user: GetUserDataResponse;
if (!PNID || !userSettings || !bodyCheck.success) {
try {
user = await getUserAccountData(request.pid);
} catch (error) {
// TODO - Log this error
response.sendStatus(403);
return;
}
const communityID: string = bodyCheck.data.community_id;
if (!user.mii) {
// * This should never happen, but TypeScript complains so check anyway
// TODO - Better errors
response.status(422);
return;
}
const userSettings: HydratedSettingsDocument | null = await getUserSettings(request.pid);
const bodyCheck = newPostSchema.safeParse(request.body);
if (!userSettings || !bodyCheck.success) {
response.sendStatus(403);
return;
}
const communityID: string | undefined = bodyCheck.data.community_id || '';
let messageBody: string = bodyCheck.data.body;
const painting: string = bodyCheck.data.painting?.replace(/\0/g, '').trim() || '';
const screenshot: string = bodyCheck.data.screenshot?.replace(/\0/g, '').trim() || '';
const appData: string = bodyCheck.data.app_data?.replace(/[^A-Za-z0-9+/=\s]/g, '').trim() || '';
const feelingID: number = parseInt(bodyCheck.data.feeling_id);
const searchKey: string[] = bodyCheck.data.search_key;
const topicTag: string = bodyCheck.data.topic_tag;
const searchKey: string[] | undefined = bodyCheck.data.search_key || [];
const topicTag: string | undefined = bodyCheck.data.topic_tag || '';
const autopost: string = bodyCheck.data.is_autopost;
const spoiler: string | undefined = bodyCheck.data.spoiler;
const jumpable: string = bodyCheck.data.is_app_jumpable;
const spoiler: string | undefined = bodyCheck.data.is_spoiler;
const jumpable: string | undefined = bodyCheck.data.is_app_jumpable;
const languageID: number = parseInt(bodyCheck.data.language_id);
const countryID: number = parseInt(request.paramPack.country_id);
const platformID: number = parseInt(request.paramPack.platform_id);
@ -330,12 +345,12 @@ async function newPost(request: express.Request, response: express.Response): Pr
is_spoiler: (spoiler) ? 1 : 0,
is_app_jumpable: (jumpable) ? 1 : 0,
language_id: languageID,
mii: PNID.mii.data,
mii_face_url: `https://mii.olv.pretendo.cc/mii/${PNID.pid}/${miiFace}`,
mii: user.mii.data,
mii_face_url: `https://mii.olv.pretendo.cc/mii/${user.pid}/${miiFace}`,
pid: request.pid,
platform_id: platformID,
region_id: regionID,
verified: (PNID.access_level === 2 || PNID.access_level === 3),
verified: (user.accessLevel === 2 || user.accessLevel === 3),
parent: parentPost ? parentPost.id : null,
removed: false
};

18
src/services/api/routes/topics.ts
View File

@ -2,10 +2,11 @@ import express from 'express';
import memoize from 'memoizee';
import moment from 'moment';
import xmlbuilder from 'xmlbuilder';
import { getPNID, getEndpoint, getPostsBytitleID } from '@/database';
import { GetUserDataResponse } from 'pretendo-grpc-ts/dist/account/get_user_data_rpc';
import { getUserAccountData } from '@/util';
import { getEndpoint, getPostsBytitleID } from '@/database';
import { Post } from '@/models/post';
import { Community } from '@/models/community';
import { HydratedPNIDDocument } from '@/types/mongoose/pnid';
import { HydratedEndpointDocument } from '@/types/mongoose/endpoint';
import { HydratedCommunityDocument } from '@/types/mongoose/community';
import { HydratedPostDocument } from '@/types/mongoose/post';
@ -22,11 +23,20 @@ const memoizedGenerateTopicsXML = memoize(generateTopicsXML, {
router.get('/', async function (request: express.Request, response: express.Response): Promise<void> {
response.type('application/xml');
const user: HydratedPNIDDocument | null = await getPNID(request.pid);
let user: GetUserDataResponse;
try {
user = await getUserAccountData(request.pid);
} catch (error) {
// TODO - Log this error
response.sendStatus(403);
return;
}
let discovery: HydratedEndpointDocument | null;
if (user) {
discovery = await getEndpoint(user.server_access_level);
discovery = await getEndpoint(user.serverAccessLevel);
} else {
discovery = await getEndpoint('prod');
}

17
src/services/discovery/routes/discovery.ts
View File

@ -1,7 +1,8 @@
import express from 'express';
import xmlbuilder from 'xmlbuilder';
import { getPNID, getEndpoint } from '@/database';
import { HydratedPNIDDocument } from '@/types/mongoose/pnid';
import { GetUserDataResponse } from 'pretendo-grpc-ts/dist/account/get_user_data_rpc';
import { getUserAccountData } from '@/util';
import { getEndpoint } from '@/database';
import { HydratedEndpointDocument } from '@/types/mongoose/endpoint';
const router: express.Router = express.Router();
@ -10,12 +11,20 @@ const router: express.Router = express.Router();
router.get('/', async function (request: express.Request, response: express.Response): Promise<void> {
response.type('application/xml');
const user: HydratedPNIDDocument | null = await getPNID(request.pid);
let user: GetUserDataResponse;
try {
user = await getUserAccountData(request.pid);
} catch (error) {
// TODO - Log this error
response.sendStatus(404);
return;
}
let discovery: HydratedEndpointDocument | null;
if (user) {
discovery = await getEndpoint(user.server_access_level);
discovery = await getEndpoint(user.serverAccessLevel);
} else {
discovery = await getEndpoint('prod');
}

9
src/types/common/config.ts
View File

@ -10,10 +10,6 @@ export interface Config {
connection_string: string;
options: mongoose.ConnectOptions;
};
account_db: {
connection_string: string;
options: mongoose.ConnectOptions;
};
s3: {
endpoint: string;
key: string;
@ -25,5 +21,10 @@ export interface Config {
port: number;
api_key: string;
};
account: {
ip: string;
port: number;
api_key: string;
};
};
}

42
src/types/mongoose/pnid.ts
View File

@ -1,42 +0,0 @@
import { Model, HydratedDocument } from 'mongoose';
enum ACCESS_LEVEL {
Banned = -1,
Standard = 0,
Tester = 1,
Mod = 2,
Developer = 3
}
type SERVER_ACCESS_LEVEL = 'prod' | 'test' | 'dev';
export interface IPNID {
access_level: ACCESS_LEVEL;
server_access_level: SERVER_ACCESS_LEVEL;
pid: number;
username: string;
birthdate: string;
country: string;
mii: {
name: string;
data: string;
};
connections: {
stripe: {
customer_id: string;
subscription_id: string;
price_id: string;
tier_level: number;
tier_name: string;
latest_webhook_timestamp: number;
};
};
}
export interface IPNIDMethods {}
interface IPNIDQueryHelpers {}
export interface PNIDModel extends Model<IPNID, IPNIDQueryHelpers, IPNIDMethods> {}
export type HydratedPNIDDocument = HydratedDocument<IPNID, IPNIDMethods>

29
src/util.ts
View File

@ -19,10 +19,15 @@ import { GetUserFriendPIDsResponse } from 'pretendo-grpc-ts/dist/friends/get_use
import { GetUserFriendRequestsIncomingResponse } from 'pretendo-grpc-ts/dist/friends/get_user_friend_requests_incoming_rpc';
import { FriendRequest } from 'pretendo-grpc-ts/dist/friends/friend_request';
const { ip, port, api_key } = config.grpc.friends;
import { AccountClient, AccountDefinition } from 'pretendo-grpc-ts/dist/account/account_service';
import { GetUserDataResponse } from 'pretendo-grpc-ts/dist/account/get_user_data_rpc';
const gRPCChannel = createChannel(`${ip}:${port}`); // * nice-grpc doesn't export ChannelImplementation so this can't be typed
const gRPCFriendsClient: FriendsClient = createClient(FriendsDefinition, gRPCChannel);
// * nice-grpc doesn't export ChannelImplementation so this can't be typed
const gRPCFriendsChannel = createChannel(`${config.grpc.friends.ip}:${config.grpc.friends.port}`);
const gRPCFriendsClient: FriendsClient = createClient(FriendsDefinition, gRPCFriendsChannel);
const gRPCAccountChannel = createChannel(`${config.grpc.account.ip}:${config.grpc.account.port}`);
const gRPCAccountClient: AccountClient = createClient(AccountDefinition, gRPCAccountChannel);
const s3: aws.S3 = new aws.S3({
endpoint: new aws.Endpoint(config.s3.endpoint),
@ -167,7 +172,7 @@ export async function getUserFriendPIDs(pid: number): Promise<number[]> {
pid: pid
}, {
metadata: Metadata({
'X-API-Key': api_key
'X-API-Key': config.grpc.friends.api_key
})
});
@ -175,15 +180,25 @@ export async function getUserFriendPIDs(pid: number): Promise<number[]> {
}
export async function getUserFriendRequestsIncoming(pid: number): Promise<FriendRequest[]> {
const requests: GetUserFriendRequestsIncomingResponse = await gRPCFriendsClient.getUserFriendRequestsIncoming({
const response: GetUserFriendRequestsIncomingResponse = await gRPCFriendsClient.getUserFriendRequestsIncoming({
pid: pid
}, {
metadata: Metadata({
'X-API-Key': api_key
'X-API-Key': config.grpc.friends.api_key
})
});
return requests.friendRequests;
return response.friendRequests;
}
export function getUserAccountData(pid: number): Promise<GetUserDataResponse> {
return gRPCAccountClient.getUserData({
pid: pid
}, {
metadata: Metadata({
'X-API-Key': config.grpc.account.api_key
})
});
}
export function makeSafeQs(query: ParsedQs): SafeQs {