Patrons with node cron

.eslintrc.js

@@ -22,6 +22,7 @@ module.exports = {
     "no-constant-condition": ["error", { checkLoops: false }],
     "no-console": ["error", { allow: ["warn", "error"] }],
     "no-warning-comments": ["warn", { terms: ["xxx"] }],
+    "no-var": 0,
     "@typescript-eslint/no-unsafe-return": 0,
     "@typescript-eslint/no-unsafe-member-access": 0,
     "@typescript-eslint/no-unsafe-assignment": 0,

app/entry.server.tsx

@@ -1,6 +1,8 @@
 import type { EntryContext } from "@remix-run/node";
 import { RemixServer } from "@remix-run/react";
 import { renderToString } from "react-dom/server";
+import cron from "node-cron";
+import { updatePatreonData } from "./modules/patreon";
 
 export default function handleRequest(
   request: Request,
@@ -19,3 +21,19 @@ export default function handleRequest(
     headers: responseHeaders,
   });
 }
+
+// example from https://github.com/BenMcH/remix-rss/blob/main/app/entry.server.tsx
+declare global {
+  var appStartSignal: undefined | true;
+}
+
+if (!global.appStartSignal && process.env.NODE_ENV === "production") {
+  global.appStartSignal = true;
+
+  // every 2 hours
+  cron.schedule(
+    "0 */2 * * *",
+    // @ts-expect-error seems to be mistyped
+    updatePatreonData().catch((err) => console.error(err))
+  );
+}

app/modules/patreon/constants.ts

@@ -0,0 +1,8 @@
+export const PATREON_INITIAL_URL =
+  "https://www.patreon.com/api/oauth2/api/campaigns/2744004/pledges?include=patron.null,reward.null";
+
+// tier 1 lowest, tier 4 highest
+export const TIER_1_ID = "6959473";
+export const TIER_2_ID = "6381152";
+export const TIER_3_ID = "6381153";
+export const TIER_4_ID = "6959564";

app/modules/patreon/index.ts

@@ -0,0 +1 @@
+export { updatePatreonData } from "./updater";

app/modules/patreon/schema.ts

@@ -0,0 +1,40 @@
+import { z } from "zod";
+import { TIER_1_ID, TIER_2_ID, TIER_3_ID, TIER_4_ID } from "./constants";
+
+export const patronResponseSchema = z.object({
+  data: z.array(
+    z.object({
+      attributes: z.object({
+        declined_since: z.string().nullable(),
+        created_at: z.string(),
+      }),
+      relationships: z.object({
+        patron: z.object({ data: z.object({ id: z.string() }) }),
+        reward: z.object({
+          data: z.object({
+            id: z.enum([TIER_1_ID, TIER_2_ID, TIER_3_ID, TIER_4_ID]),
+          }),
+        }),
+      }),
+    })
+  ),
+  included: z.array(
+    z.discriminatedUnion("type", [
+      z.object({
+        type: z.literal("user"),
+        id: z.string(),
+        attributes: z.object({
+          email: z.string(),
+          full_name: z.string(),
+          social_connections: z.object({
+            discord: z.object({ user_id: z.string() }).nullable(),
+          }),
+        }),
+      }),
+      z.object({ type: z.literal("reward") }),
+      z.object({ type: z.literal("goal") }),
+      z.object({ type: z.literal("campaign") }),
+    ])
+  ),
+  links: z.object({ next: z.string().nullish() }),
+});

app/modules/patreon/updater.ts

@@ -0,0 +1,145 @@
+import type { z } from "zod";
+import { db } from "~/db";
+import type { UpdatePatronDataArgs } from "~/db/models/users.server";
+import { dateToDatabaseTimestamp } from "~/utils/dates";
+import { fetchWithTimeout } from "~/utils/fetch";
+import type { Unpacked } from "~/utils/types";
+import {
+  PATREON_INITIAL_URL,
+  TIER_1_ID,
+  TIER_2_ID,
+  TIER_3_ID,
+  TIER_4_ID,
+} from "./constants";
+import { patronResponseSchema } from "./schema";
+
+interface NoDiscordConnectionUser {
+  email: string;
+  name: string;
+}
+
+export async function updatePatreonData(): Promise<void> {
+  const patrons: UpdatePatronDataArgs = [];
+  const noDiscordConnected: Array<NoDiscordConnectionUser> = [];
+  const noDataIds: Array<string> = [];
+  let nextUrlToFetchWith = PATREON_INITIAL_URL;
+
+  while (nextUrlToFetchWith) {
+    const patronData = await fetchPatronData(nextUrlToFetchWith);
+
+    const parsed = parsePatronData(patronData);
+    patrons.push(...parsed.patrons);
+    noDiscordConnected.push(...parsed.noDiscordConnection);
+    noDataIds.push(...parsed.noDataIds);
+
+    // TS freaks out if we don't keep nextUrlToFetchWith string so that's why this weird thing here
+    nextUrlToFetchWith = patronData.links.next ?? "";
+  }
+
+  db.users.updatePatronData(patrons);
+
+  // eslint-disable-next-line no-console
+  console.log(
+    `Added ${patrons.length} patrons. ${
+      noDiscordConnected.length
+    } patrons had no Discord connected. No full data for following Patreon ID's: ${noDataIds.join(
+      ", "
+    )}`
+  );
+}
+
+async function fetchPatronData(urlToFetch: string) {
+  if (!process.env["PATREON_ACCESS_TOKEN"]) {
+    throw new Error("Missing Patreon access token");
+  }
+
+  const response = await fetchWithTimeout(
+    urlToFetch,
+    {
+      headers: {
+        Authorization: `Bearer ${process.env["PATREON_ACCESS_TOKEN"]}`,
+      },
+    },
+    30_000
+  );
+
+  if (!response.ok) {
+    throw new Error(
+      `Patreon response not succesful. Status code was: ${response.status}`
+    );
+  }
+
+  return patronResponseSchema.parse(await response.json());
+}
+
+function parsePatronData({
+  data,
+  included,
+}: z.infer<typeof patronResponseSchema>) {
+  const patronsWithIds: Array<
+    {
+      patreonId: string;
+    } & Omit<Unpacked<UpdatePatronDataArgs>, "discordId">
+  > = [];
+
+  for (const patron of data) {
+    // from Patreon:
+    // "declined_since indicates the date of the most recent payment if it failed, or `null` if the most recent payment succeeded.
+    // A pledge with a non-null declined_since should be treated as invalid."
+    if (patron.attributes.declined_since) {
+      continue;
+    }
+
+    patronsWithIds.push({
+      patreonId: patron.relationships.patron.data.id,
+      patronSince: dateToDatabaseTimestamp(
+        new Date(patron.attributes.created_at)
+      ),
+      patronTier: idToTier(patron.relationships.reward.data.id),
+    });
+  }
+
+  const result: {
+    patrons: UpdatePatronDataArgs;
+    noDiscordConnection: Array<NoDiscordConnectionUser>;
+    noDataIds: string[];
+  } = {
+    patrons: [],
+    noDiscordConnection: [],
+    noDataIds: [],
+  };
+  for (const extraData of included) {
+    if (extraData.type !== "user") continue;
+
+    const patronData = patronsWithIds.find((p) => p.patreonId === extraData.id);
+    if (!patronData) {
+      result.noDataIds.push(extraData.id);
+      continue;
+    }
+
+    const discordId = extraData.attributes.social_connections.discord?.user_id;
+    if (!discordId) {
+      result.noDiscordConnection.push({
+        email: extraData.attributes.email,
+        name: extraData.attributes.full_name,
+      });
+      continue;
+    }
+
+    result.patrons.push({
+      patronSince: patronData.patronSince,
+      discordId,
+      patronTier: patronData.patronTier,
+    });
+  }
+
+  return result;
+}
+
+function idToTier(id: string) {
+  const tier = [null, TIER_1_ID, TIER_2_ID, TIER_3_ID, TIER_4_ID].indexOf(id);
+
+  if (tier === -1) throw new Error(`Invalid tier id: ${id}`);
+
+  return tier;
+}

app/routes/patrons.tsx

@@ -1,34 +1,7 @@
 import type { ActionFunction } from "@remix-run/node";
-import { z } from "zod";
-import { db } from "~/db";
-import type { UpdatePatronDataArgs } from "~/db/models/users.server";
 import { getUser } from "~/modules/auth";
+import { updatePatreonData } from "~/modules/patreon";
 import { canAccessLohiEndpoint, canPerformAdminActions } from "~/permissions";
-import { dateToDatabaseTimestamp } from "~/utils/dates";
-import { fetchWithTimeout } from "~/utils/fetch";
-import type { Unpacked } from "~/utils/types";
-
-const PATREON_INITIAL_URL =
-  "https://www.patreon.com/api/oauth2/api/campaigns/2744004/pledges?include=patron.null,reward.null";
-
-// tier 1 lowest, tier 4 highest
-const TIER_1_ID = "6959473";
-const TIER_2_ID = "6381152";
-const TIER_3_ID = "6381153";
-const TIER_4_ID = "6959564";
-
-function idToTier(id: string) {
-  const tier = [null, TIER_1_ID, TIER_2_ID, TIER_3_ID, TIER_4_ID].indexOf(id);
-
-  if (tier === -1) throw new Error(`Invalid tier id: ${id}`);
-
-  return tier;
-}
-
-interface NoDiscordConnectionUser {
-  email: string;
-  name: string;
-}
 
 export const action: ActionFunction = async ({ request }) => {
   const user = await getUser(request);
@@ -37,159 +10,7 @@ export const action: ActionFunction = async ({ request }) => {
     throw new Response("Not authorized", { status: 403 });
   }
 
-  const patrons: UpdatePatronDataArgs = [];
-  const noDiscordConnected: Array<NoDiscordConnectionUser> = [];
-  const noDataIds: Array<string> = [];
-  let nextUrlToFetchWith: string | undefined;
+  await updatePatreonData();
 
-  while (true) {
-    const patronData = await fetchPatronData(nextUrlToFetchWith);
-
-    const parsed = parsePatronData(patronData);
-    patrons.push(...parsed.patrons);
-    noDiscordConnected.push(...parsed.noDiscordConnection);
-    noDataIds.push(...parsed.noDataIds);
-
-    if (!patronData.links.next) break;
-
-    nextUrlToFetchWith = patronData.links.next;
-  }
-
-  db.users.updatePatronData(patrons);
-
-  return new Response(
-    `Added ${patrons.length} patrons. ${
-      noDiscordConnected.length
-    } patrons had no Discord connected. No full data for following Patreon ID's: ${noDataIds.join(
-      ", "
-    )}`,
-    { status: 200 }
-  );
+  return null;
 };
-
-const patronResponseSchema = z.object({
-  data: z.array(
-    z.object({
-      attributes: z.object({
-        declined_since: z.string().nullable(),
-        created_at: z.string(),
-      }),
-      relationships: z.object({
-        patron: z.object({ data: z.object({ id: z.string() }) }),
-        reward: z.object({
-          data: z.object({
-            id: z.enum([TIER_1_ID, TIER_2_ID, TIER_3_ID, TIER_4_ID]),
-          }),
-        }),
-      }),
-    })
-  ),
-  included: z.array(
-    z.discriminatedUnion("type", [
-      z.object({
-        type: z.literal("user"),
-        id: z.string(),
-        attributes: z.object({
-          email: z.string(),
-          full_name: z.string(),
-          social_connections: z.object({
-            discord: z.object({ user_id: z.string() }).nullable(),
-          }),
-        }),
-      }),
-      z.object({ type: z.literal("reward") }),
-      z.object({ type: z.literal("goal") }),
-      z.object({ type: z.literal("campaign") }),
-    ])
-  ),
-  links: z.object({ next: z.string().nullish() }),
-});
-
-async function fetchPatronData(nextUrl?: string) {
-  if (!process.env["PATREON_ACCESS_TOKEN"]) {
-    throw new Response("Missing Patreon access token", { status: 500 });
-  }
-
-  const response = await fetchWithTimeout(
-    nextUrl ?? PATREON_INITIAL_URL,
-    {
-      headers: {
-        Authorization: `Bearer ${process.env["PATREON_ACCESS_TOKEN"]}`,
-      },
-    },
-    30_000
-  );
-
-  if (!response.ok) {
-    throw new Response(
-      `Patreon response not succesful. Status code was: ${response.status}`,
-      { status: 502 }
-    );
-  }
-
-  return patronResponseSchema.parse(await response.json());
-}
-
-function parsePatronData({
-  data,
-  included,
-}: z.infer<typeof patronResponseSchema>) {
-  const patronsWithIds: Array<
-    {
-      patreonId: string;
-    } & Omit<Unpacked<UpdatePatronDataArgs>, "discordId">
-  > = [];
-
-  for (const patron of data) {
-    // from Patreon:
-    // "declined_since indicates the date of the most recent payment if it failed, or `null` if the most recent payment succeeded.
-    // A pledge with a non-null declined_since should be treated as invalid."
-    if (patron.attributes.declined_since) {
-      continue;
-    }
-
-    patronsWithIds.push({
-      patreonId: patron.relationships.patron.data.id,
-      patronSince: dateToDatabaseTimestamp(
-        new Date(patron.attributes.created_at)
-      ),
-      patronTier: idToTier(patron.relationships.reward.data.id),
-    });
-  }
-
-  const result: {
-    patrons: UpdatePatronDataArgs;
-    noDiscordConnection: Array<NoDiscordConnectionUser>;
-    noDataIds: string[];
-  } = {
-    patrons: [],
-    noDiscordConnection: [],
-    noDataIds: [],
-  };
-  for (const extraData of included) {
-    if (extraData.type !== "user") continue;
-
-    const patronData = patronsWithIds.find((p) => p.patreonId === extraData.id);
-    if (!patronData) {
-      result.noDataIds.push(extraData.id);
-      continue;
-    }
-
-    const discordId = extraData.attributes.social_connections.discord?.user_id;
-    if (!discordId) {
-      result.noDiscordConnection.push({
-        email: extraData.attributes.email,
-        name: extraData.attributes.full_name,
-      });
-      continue;
-    }
-
-    result.patrons.push({
-      patronSince: patronData.patronSince,
-      discordId,
-      patronTier: patronData.patronTier,
-    });
-  }
-
-  return result;
-}

package-lock.json

@@ -18,6 +18,7 @@
         "date-fns": "^2.28.0",
         "fuse.js": "^6.6.2",
         "just-shuffle": "^4.0.1",
+        "node-cron": "3.0.0",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
         "react-popper": "^2.3.0",
@@ -30,6 +31,7 @@
         "@remix-run/dev": "^1.6.4",
         "@remix-run/eslint-config": "^1.6.4",
         "@types/better-sqlite3": "^7.5.0",
+        "@types/node-cron": "^3.0.2",
         "@types/react": "^18.0.15",
         "@types/react-dom": "^18.0.6",
         "@typescript-eslint/eslint-plugin": "^5.30.5",
@@ -3156,6 +3158,12 @@
       "integrity": "sha512-AR0x5HbXGqkEx9CadRH3EBYx/VkiUgZIhP4wvPn/+5KIsgpNoyFaRlVe0Zlx9gRtg8fA06a9tskE2MSN7TcG4Q==",
       "dev": true
     },
+    "node_modules/@types/node-cron": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.2.tgz",
+      "integrity": "sha512-SK/4GYWiWvGLPl/yv+Tm5oLYbzMx1V3y7CsNTvOb3vF8O9oXH11U6/zckISHnBl4YH8MvXHFIUXbYoBONSdmzw==",
+      "dev": true
+    },
     "node_modules/@types/normalize-package-data": {
       "version": "2.4.1",
       "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz",
@@ -11105,6 +11113,25 @@
       "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
       "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
     },
+    "node_modules/moment": {
+      "version": "2.29.4",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
+      "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==",
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/moment-timezone": {
+      "version": "0.5.34",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.34.tgz",
+      "integrity": "sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg==",
+      "dependencies": {
+        "moment": ">= 2.9.0"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/morgan": {
       "version": "1.10.0",
       "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.0.tgz",
@@ -11236,6 +11263,17 @@
       "dev": true,
       "optional": true
     },
+    "node_modules/node-cron": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.0.tgz",
+      "integrity": "sha512-DDwIvvuCwrNiaU7HEivFDULcaQualDv7KoNlB/UU1wPW0n1tDEmBJKhEIE6DlF2FuoOHcNbLJ8ITL2Iv/3AWmA==",
+      "dependencies": {
+        "moment-timezone": "^0.5.31"
+      },
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
     "node_modules/node-dir": {
       "version": "0.1.17",
       "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz",
@@ -17776,6 +17814,12 @@
       "integrity": "sha512-AR0x5HbXGqkEx9CadRH3EBYx/VkiUgZIhP4wvPn/+5KIsgpNoyFaRlVe0Zlx9gRtg8fA06a9tskE2MSN7TcG4Q==",
       "dev": true
     },
+    "@types/node-cron": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.2.tgz",
+      "integrity": "sha512-SK/4GYWiWvGLPl/yv+Tm5oLYbzMx1V3y7CsNTvOb3vF8O9oXH11U6/zckISHnBl4YH8MvXHFIUXbYoBONSdmzw==",
+      "dev": true
+    },
     "@types/normalize-package-data": {
       "version": "2.4.1",
       "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz",
@@ -23533,6 +23577,19 @@
       "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
       "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
     },
+    "moment": {
+      "version": "2.29.4",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
+      "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w=="
+    },
+    "moment-timezone": {
+      "version": "0.5.34",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.34.tgz",
+      "integrity": "sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg==",
+      "requires": {
+        "moment": ">= 2.9.0"
+      }
+    },
     "morgan": {
       "version": "1.10.0",
       "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.0.tgz",
@@ -23639,6 +23696,14 @@
       "dev": true,
       "optional": true
     },
+    "node-cron": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.0.tgz",
+      "integrity": "sha512-DDwIvvuCwrNiaU7HEivFDULcaQualDv7KoNlB/UU1wPW0n1tDEmBJKhEIE6DlF2FuoOHcNbLJ8ITL2Iv/3AWmA==",
+      "requires": {
+        "moment-timezone": "^0.5.31"
+      }
+    },
     "node-dir": {
       "version": "0.1.17",
       "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz",

package.json

@@ -39,6 +39,7 @@
     "date-fns": "^2.28.0",
     "fuse.js": "^6.6.2",
     "just-shuffle": "^4.0.1",
+    "node-cron": "3.0.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-popper": "^2.3.0",
@@ -51,6 +52,7 @@
     "@remix-run/dev": "^1.6.4",
     "@remix-run/eslint-config": "^1.6.4",
     "@types/better-sqlite3": "^7.5.0",
+    "@types/node-cron": "^3.0.2",
     "@types/react": "^18.0.15",
     "@types/react-dom": "^18.0.6",
     "@typescript-eslint/eslint-plugin": "^5.30.5",