diff --git a/.eslintrc.js b/.eslintrc.js index 6a890f0f4..6e251d069 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -22,6 +22,7 @@ module.exports = { "no-constant-condition": ["error", { checkLoops: false }], "no-console": ["error", { allow: ["warn", "error"] }], "no-warning-comments": ["warn", { terms: ["xxx"] }], + "no-var": 0, "@typescript-eslint/no-unsafe-return": 0, "@typescript-eslint/no-unsafe-member-access": 0, "@typescript-eslint/no-unsafe-assignment": 0, diff --git a/app/entry.server.tsx b/app/entry.server.tsx index 5afa18235..a4d5a6016 100644 --- a/app/entry.server.tsx +++ b/app/entry.server.tsx @@ -1,6 +1,8 @@ import type { EntryContext } from "@remix-run/node"; import { RemixServer } from "@remix-run/react"; import { renderToString } from "react-dom/server"; +import cron from "node-cron"; +import { updatePatreonData } from "./modules/patreon"; export default function handleRequest( request: Request, @@ -19,3 +21,19 @@ export default function handleRequest( headers: responseHeaders, }); } + +// example from https://github.com/BenMcH/remix-rss/blob/main/app/entry.server.tsx +declare global { + var appStartSignal: undefined | true; +} + +if (!global.appStartSignal && process.env.NODE_ENV === "production") { + global.appStartSignal = true; + + // every 2 hours + cron.schedule( + "0 */2 * * *", + // @ts-expect-error seems to be mistyped + updatePatreonData().catch((err) => console.error(err)) + ); +} diff --git a/app/modules/patreon/constants.ts b/app/modules/patreon/constants.ts new file mode 100644 index 000000000..3a0f4fa1e --- /dev/null +++ b/app/modules/patreon/constants.ts @@ -0,0 +1,8 @@ +export const PATREON_INITIAL_URL = + "https://www.patreon.com/api/oauth2/api/campaigns/2744004/pledges?include=patron.null,reward.null"; + +// tier 1 lowest, tier 4 highest +export const TIER_1_ID = "6959473"; +export const TIER_2_ID = "6381152"; +export const TIER_3_ID = "6381153"; +export const TIER_4_ID = "6959564"; diff --git a/app/modules/patreon/index.ts b/app/modules/patreon/index.ts new file mode 100644 index 000000000..207582d21 --- /dev/null +++ b/app/modules/patreon/index.ts @@ -0,0 +1 @@ +export { updatePatreonData } from "./updater"; diff --git a/app/modules/patreon/schema.ts b/app/modules/patreon/schema.ts new file mode 100644 index 000000000..4a98b9096 --- /dev/null +++ b/app/modules/patreon/schema.ts @@ -0,0 +1,40 @@ +import { z } from "zod"; +import { TIER_1_ID, TIER_2_ID, TIER_3_ID, TIER_4_ID } from "./constants"; + +export const patronResponseSchema = z.object({ + data: z.array( + z.object({ + attributes: z.object({ + declined_since: z.string().nullable(), + created_at: z.string(), + }), + relationships: z.object({ + patron: z.object({ data: z.object({ id: z.string() }) }), + reward: z.object({ + data: z.object({ + id: z.enum([TIER_1_ID, TIER_2_ID, TIER_3_ID, TIER_4_ID]), + }), + }), + }), + }) + ), + included: z.array( + z.discriminatedUnion("type", [ + z.object({ + type: z.literal("user"), + id: z.string(), + attributes: z.object({ + email: z.string(), + full_name: z.string(), + social_connections: z.object({ + discord: z.object({ user_id: z.string() }).nullable(), + }), + }), + }), + z.object({ type: z.literal("reward") }), + z.object({ type: z.literal("goal") }), + z.object({ type: z.literal("campaign") }), + ]) + ), + links: z.object({ next: z.string().nullish() }), +}); diff --git a/app/modules/patreon/updater.ts b/app/modules/patreon/updater.ts new file mode 100644 index 000000000..7e3c919db --- /dev/null +++ b/app/modules/patreon/updater.ts @@ -0,0 +1,145 @@ +import type { z } from "zod"; +import { db } from "~/db"; +import type { UpdatePatronDataArgs } from "~/db/models/users.server"; +import { dateToDatabaseTimestamp } from "~/utils/dates"; +import { fetchWithTimeout } from "~/utils/fetch"; +import type { Unpacked } from "~/utils/types"; +import { + PATREON_INITIAL_URL, + TIER_1_ID, + TIER_2_ID, + TIER_3_ID, + TIER_4_ID, +} from "./constants"; +import { patronResponseSchema } from "./schema"; + +interface NoDiscordConnectionUser { + email: string; + name: string; +} + +export async function updatePatreonData(): Promise { + const patrons: UpdatePatronDataArgs = []; + const noDiscordConnected: Array = []; + const noDataIds: Array = []; + let nextUrlToFetchWith = PATREON_INITIAL_URL; + + while (nextUrlToFetchWith) { + const patronData = await fetchPatronData(nextUrlToFetchWith); + + const parsed = parsePatronData(patronData); + patrons.push(...parsed.patrons); + noDiscordConnected.push(...parsed.noDiscordConnection); + noDataIds.push(...parsed.noDataIds); + + // TS freaks out if we don't keep nextUrlToFetchWith string so that's why this weird thing here + nextUrlToFetchWith = patronData.links.next ?? ""; + } + + db.users.updatePatronData(patrons); + + // eslint-disable-next-line no-console + console.log( + `Added ${patrons.length} patrons. ${ + noDiscordConnected.length + } patrons had no Discord connected. No full data for following Patreon ID's: ${noDataIds.join( + ", " + )}` + ); +} + +async function fetchPatronData(urlToFetch: string) { + if (!process.env["PATREON_ACCESS_TOKEN"]) { + throw new Error("Missing Patreon access token"); + } + + const response = await fetchWithTimeout( + urlToFetch, + { + headers: { + Authorization: `Bearer ${process.env["PATREON_ACCESS_TOKEN"]}`, + }, + }, + 30_000 + ); + + if (!response.ok) { + throw new Error( + `Patreon response not succesful. Status code was: ${response.status}` + ); + } + + return patronResponseSchema.parse(await response.json()); +} + +function parsePatronData({ + data, + included, +}: z.infer) { + const patronsWithIds: Array< + { + patreonId: string; + } & Omit, "discordId"> + > = []; + + for (const patron of data) { + // from Patreon: + // "declined_since indicates the date of the most recent payment if it failed, or `null` if the most recent payment succeeded. + // A pledge with a non-null declined_since should be treated as invalid." + if (patron.attributes.declined_since) { + continue; + } + + patronsWithIds.push({ + patreonId: patron.relationships.patron.data.id, + patronSince: dateToDatabaseTimestamp( + new Date(patron.attributes.created_at) + ), + patronTier: idToTier(patron.relationships.reward.data.id), + }); + } + + const result: { + patrons: UpdatePatronDataArgs; + noDiscordConnection: Array; + noDataIds: string[]; + } = { + patrons: [], + noDiscordConnection: [], + noDataIds: [], + }; + for (const extraData of included) { + if (extraData.type !== "user") continue; + + const patronData = patronsWithIds.find((p) => p.patreonId === extraData.id); + if (!patronData) { + result.noDataIds.push(extraData.id); + continue; + } + + const discordId = extraData.attributes.social_connections.discord?.user_id; + if (!discordId) { + result.noDiscordConnection.push({ + email: extraData.attributes.email, + name: extraData.attributes.full_name, + }); + continue; + } + + result.patrons.push({ + patronSince: patronData.patronSince, + discordId, + patronTier: patronData.patronTier, + }); + } + + return result; +} + +function idToTier(id: string) { + const tier = [null, TIER_1_ID, TIER_2_ID, TIER_3_ID, TIER_4_ID].indexOf(id); + + if (tier === -1) throw new Error(`Invalid tier id: ${id}`); + + return tier; +} diff --git a/app/routes/patrons.tsx b/app/routes/patrons.tsx index c360b6d8a..3845a25de 100644 --- a/app/routes/patrons.tsx +++ b/app/routes/patrons.tsx @@ -1,34 +1,7 @@ import type { ActionFunction } from "@remix-run/node"; -import { z } from "zod"; -import { db } from "~/db"; -import type { UpdatePatronDataArgs } from "~/db/models/users.server"; import { getUser } from "~/modules/auth"; +import { updatePatreonData } from "~/modules/patreon"; import { canAccessLohiEndpoint, canPerformAdminActions } from "~/permissions"; -import { dateToDatabaseTimestamp } from "~/utils/dates"; -import { fetchWithTimeout } from "~/utils/fetch"; -import type { Unpacked } from "~/utils/types"; - -const PATREON_INITIAL_URL = - "https://www.patreon.com/api/oauth2/api/campaigns/2744004/pledges?include=patron.null,reward.null"; - -// tier 1 lowest, tier 4 highest -const TIER_1_ID = "6959473"; -const TIER_2_ID = "6381152"; -const TIER_3_ID = "6381153"; -const TIER_4_ID = "6959564"; - -function idToTier(id: string) { - const tier = [null, TIER_1_ID, TIER_2_ID, TIER_3_ID, TIER_4_ID].indexOf(id); - - if (tier === -1) throw new Error(`Invalid tier id: ${id}`); - - return tier; -} - -interface NoDiscordConnectionUser { - email: string; - name: string; -} export const action: ActionFunction = async ({ request }) => { const user = await getUser(request); @@ -37,159 +10,7 @@ export const action: ActionFunction = async ({ request }) => { throw new Response("Not authorized", { status: 403 }); } - const patrons: UpdatePatronDataArgs = []; - const noDiscordConnected: Array = []; - const noDataIds: Array = []; - let nextUrlToFetchWith: string | undefined; + await updatePatreonData(); - while (true) { - const patronData = await fetchPatronData(nextUrlToFetchWith); - - const parsed = parsePatronData(patronData); - patrons.push(...parsed.patrons); - noDiscordConnected.push(...parsed.noDiscordConnection); - noDataIds.push(...parsed.noDataIds); - - if (!patronData.links.next) break; - - nextUrlToFetchWith = patronData.links.next; - } - - db.users.updatePatronData(patrons); - - return new Response( - `Added ${patrons.length} patrons. ${ - noDiscordConnected.length - } patrons had no Discord connected. No full data for following Patreon ID's: ${noDataIds.join( - ", " - )}`, - { status: 200 } - ); + return null; }; - -const patronResponseSchema = z.object({ - data: z.array( - z.object({ - attributes: z.object({ - declined_since: z.string().nullable(), - created_at: z.string(), - }), - relationships: z.object({ - patron: z.object({ data: z.object({ id: z.string() }) }), - reward: z.object({ - data: z.object({ - id: z.enum([TIER_1_ID, TIER_2_ID, TIER_3_ID, TIER_4_ID]), - }), - }), - }), - }) - ), - included: z.array( - z.discriminatedUnion("type", [ - z.object({ - type: z.literal("user"), - id: z.string(), - attributes: z.object({ - email: z.string(), - full_name: z.string(), - social_connections: z.object({ - discord: z.object({ user_id: z.string() }).nullable(), - }), - }), - }), - z.object({ type: z.literal("reward") }), - z.object({ type: z.literal("goal") }), - z.object({ type: z.literal("campaign") }), - ]) - ), - links: z.object({ next: z.string().nullish() }), -}); - -async function fetchPatronData(nextUrl?: string) { - if (!process.env["PATREON_ACCESS_TOKEN"]) { - throw new Response("Missing Patreon access token", { status: 500 }); - } - - const response = await fetchWithTimeout( - nextUrl ?? PATREON_INITIAL_URL, - { - headers: { - Authorization: `Bearer ${process.env["PATREON_ACCESS_TOKEN"]}`, - }, - }, - 30_000 - ); - - if (!response.ok) { - throw new Response( - `Patreon response not succesful. Status code was: ${response.status}`, - { status: 502 } - ); - } - - return patronResponseSchema.parse(await response.json()); -} - -function parsePatronData({ - data, - included, -}: z.infer) { - const patronsWithIds: Array< - { - patreonId: string; - } & Omit, "discordId"> - > = []; - - for (const patron of data) { - // from Patreon: - // "declined_since indicates the date of the most recent payment if it failed, or `null` if the most recent payment succeeded. - // A pledge with a non-null declined_since should be treated as invalid." - if (patron.attributes.declined_since) { - continue; - } - - patronsWithIds.push({ - patreonId: patron.relationships.patron.data.id, - patronSince: dateToDatabaseTimestamp( - new Date(patron.attributes.created_at) - ), - patronTier: idToTier(patron.relationships.reward.data.id), - }); - } - - const result: { - patrons: UpdatePatronDataArgs; - noDiscordConnection: Array; - noDataIds: string[]; - } = { - patrons: [], - noDiscordConnection: [], - noDataIds: [], - }; - for (const extraData of included) { - if (extraData.type !== "user") continue; - - const patronData = patronsWithIds.find((p) => p.patreonId === extraData.id); - if (!patronData) { - result.noDataIds.push(extraData.id); - continue; - } - - const discordId = extraData.attributes.social_connections.discord?.user_id; - if (!discordId) { - result.noDiscordConnection.push({ - email: extraData.attributes.email, - name: extraData.attributes.full_name, - }); - continue; - } - - result.patrons.push({ - patronSince: patronData.patronSince, - discordId, - patronTier: patronData.patronTier, - }); - } - - return result; -} diff --git a/package-lock.json b/package-lock.json index f0698f767..6cdbaba96 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "date-fns": "^2.28.0", "fuse.js": "^6.6.2", "just-shuffle": "^4.0.1", + "node-cron": "3.0.0", "react": "^18.2.0", "react-dom": "^18.2.0", "react-popper": "^2.3.0", @@ -30,6 +31,7 @@ "@remix-run/dev": "^1.6.4", "@remix-run/eslint-config": "^1.6.4", "@types/better-sqlite3": "^7.5.0", + "@types/node-cron": "^3.0.2", "@types/react": "^18.0.15", "@types/react-dom": "^18.0.6", "@typescript-eslint/eslint-plugin": "^5.30.5", @@ -3156,6 +3158,12 @@ "integrity": "sha512-AR0x5HbXGqkEx9CadRH3EBYx/VkiUgZIhP4wvPn/+5KIsgpNoyFaRlVe0Zlx9gRtg8fA06a9tskE2MSN7TcG4Q==", "dev": true }, + "node_modules/@types/node-cron": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.2.tgz", + "integrity": "sha512-SK/4GYWiWvGLPl/yv+Tm5oLYbzMx1V3y7CsNTvOb3vF8O9oXH11U6/zckISHnBl4YH8MvXHFIUXbYoBONSdmzw==", + "dev": true + }, "node_modules/@types/normalize-package-data": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz", @@ -11105,6 +11113,25 @@ "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, + "node_modules/moment": { + "version": "2.29.4", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", + "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==", + "engines": { + "node": "*" + } + }, + "node_modules/moment-timezone": { + "version": "0.5.34", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.34.tgz", + "integrity": "sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg==", + "dependencies": { + "moment": ">= 2.9.0" + }, + "engines": { + "node": "*" + } + }, "node_modules/morgan": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.0.tgz", @@ -11236,6 +11263,17 @@ "dev": true, "optional": true }, + "node_modules/node-cron": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.0.tgz", + "integrity": "sha512-DDwIvvuCwrNiaU7HEivFDULcaQualDv7KoNlB/UU1wPW0n1tDEmBJKhEIE6DlF2FuoOHcNbLJ8ITL2Iv/3AWmA==", + "dependencies": { + "moment-timezone": "^0.5.31" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/node-dir": { "version": "0.1.17", "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", @@ -17776,6 +17814,12 @@ "integrity": "sha512-AR0x5HbXGqkEx9CadRH3EBYx/VkiUgZIhP4wvPn/+5KIsgpNoyFaRlVe0Zlx9gRtg8fA06a9tskE2MSN7TcG4Q==", "dev": true }, + "@types/node-cron": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.2.tgz", + "integrity": "sha512-SK/4GYWiWvGLPl/yv+Tm5oLYbzMx1V3y7CsNTvOb3vF8O9oXH11U6/zckISHnBl4YH8MvXHFIUXbYoBONSdmzw==", + "dev": true + }, "@types/normalize-package-data": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz", @@ -23533,6 +23577,19 @@ "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, + "moment": { + "version": "2.29.4", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", + "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==" + }, + "moment-timezone": { + "version": "0.5.34", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.34.tgz", + "integrity": "sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg==", + "requires": { + "moment": ">= 2.9.0" + } + }, "morgan": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.0.tgz", @@ -23639,6 +23696,14 @@ "dev": true, "optional": true }, + "node-cron": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.0.tgz", + "integrity": "sha512-DDwIvvuCwrNiaU7HEivFDULcaQualDv7KoNlB/UU1wPW0n1tDEmBJKhEIE6DlF2FuoOHcNbLJ8ITL2Iv/3AWmA==", + "requires": { + "moment-timezone": "^0.5.31" + } + }, "node-dir": { "version": "0.1.17", "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", diff --git a/package.json b/package.json index f7db201ea..7e9b7a21d 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "date-fns": "^2.28.0", "fuse.js": "^6.6.2", "just-shuffle": "^4.0.1", + "node-cron": "3.0.0", "react": "^18.2.0", "react-dom": "^18.2.0", "react-popper": "^2.3.0", @@ -51,6 +52,7 @@ "@remix-run/dev": "^1.6.4", "@remix-run/eslint-config": "^1.6.4", "@types/better-sqlite3": "^7.5.0", + "@types/node-cron": "^3.0.2", "@types/react": "^18.0.15", "@types/react-dom": "^18.0.6", "@typescript-eslint/eslint-plugin": "^5.30.5",