From 072d70b42adf15ded98d5b79aa6b003f11ba3f37 Mon Sep 17 00:00:00 2001 From: Tau Date: Fri, 11 Oct 2019 11:13:17 -0400 Subject: [PATCH] Migrate to SQLite3 We'll keep the door open for side-by-side support of Postgres in the background, but due to SQLite's type system quirks we cannot use the same DDL for both databases, so we would have to maintain two sets of DDL (schema init and schema migration scripts) at once. Interested future contributors can shoulder this maintenance burden if they so choose. --- .gitignore | 1 + package.json | 4 +- schema/init/aime.sql | 22 +- schema/init/idz.sql | 118 ++++++----- schema/migrate/M0001-add-helper-funcs.sql | 81 -------- .../migrate/M0002-idz-drop-profile-ext-id.sql | 3 - schema/migrate/M0003-idz-add-team.sql | 42 ---- schema/migrate/M0004-idz-fix-col-casing.sql | 21 -- schema/migrate/M0005-merge-schemas.sql | 81 -------- schema/migrate/M0006-flatten-arrays.sql | 19 -- src/index.ts | 6 +- src/sql/index.ts | 2 +- src/sql/pg.ts | 57 ------ src/sql/sqlite.ts | 104 ++++++++++ yarn.lock | 192 +++++++----------- 15 files changed, 247 insertions(+), 506 deletions(-) delete mode 100644 schema/migrate/M0001-add-helper-funcs.sql delete mode 100644 schema/migrate/M0002-idz-drop-profile-ext-id.sql delete mode 100644 schema/migrate/M0003-idz-add-team.sql delete mode 100644 schema/migrate/M0004-idz-fix-col-casing.sql delete mode 100644 schema/migrate/M0005-merge-schemas.sql delete mode 100644 schema/migrate/M0006-flatten-arrays.sql delete mode 100644 src/sql/pg.ts create mode 100644 src/sql/sqlite.ts diff --git a/.gitignore b/.gitignore index d1b714e..e50b0a3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ bin/ +data/ node_modules/ .env diff --git a/package.json b/package.json index d1ae72e..9499d14 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "main": "src/index.js", "private": true, "dependencies": { + "better-sqlite3": "^5.4.3", "compression": "^1.7.3", "date-fns": "^1.30.1", "debug": "^4.1.1", @@ -13,12 +14,12 @@ "iconv-lite": "^0.4.24", "morgan": "^1.9.1", "multiparty": "^4.2.1", - "pg": "^7.10.0", "raw-body": "^2.3.3", "sql-bricks-postgres": "^0.5.0", "supports-color": "^7.1.0" }, "devDependencies": { + "@types/better-sqlite3": "^5.4.0", "@types/compression": "^0.0.36", "@types/debug": "^4.1.5", "@types/dotenv": "^6.1.1", @@ -26,7 +27,6 @@ "@types/jest": "^24.0.11", "@types/multiparty": "^0.0.32", "@types/node": "^11.11.3", - "@types/pg": "^7.4.14", "jest": "^24.5.0", "jest-haste-map": "^24.5.0", "jest-resolve": "^24.5.0", diff --git a/schema/init/aime.sql b/schema/init/aime.sql index 3b3d044..342185f 100644 --- a/schema/init/aime.sql +++ b/schema/init/aime.sql @@ -1,22 +1,14 @@ -create type "aime_region" as enum ( - 'JPN', - 'HKG', - 'SGP', - 'KOR', - 'USA' -); - create table "aime_shop" ( - "id" bigint primary key not null, + "id" integer primary key not null, "ext_id" integer not null, "name" text not null, - "region" "aime_region" not null, + "region" text not null, constraint "aime_shop_uq" unique ("ext_id") ); create table "aime_machine" ( - "id" bigint primary key not null, - "shop_id" bigint not null + "id" integer primary key not null, + "shop_id" integer not null references "aime_shop"("id"), "pcb_id" text not null, "keychip_id" text not null, @@ -25,15 +17,15 @@ create table "aime_machine" ( ); create table "aime_player" ( - "id" bigint primary key not null, + "id" integer primary key not null, "ext_id" integer not null, "register_time" timestamp not null, constraint "aime_player_uq" unique ("ext_id") ); create table "aime_card" ( - "id" bigint primary key not null, - "player_id" bigint not null + "id" integer primary key not null, + "player_id" integer not null references "aime_player"("id") on delete cascade, "nfc_id" text not null, diff --git a/schema/init/idz.sql b/schema/init/idz.sql index 101c23e..2bc3615 100644 --- a/schema/init/idz.sql +++ b/schema/init/idz.sql @@ -2,18 +2,14 @@ -- A "course" contains an uphill and downhill "route" -- (or inbound/outbound etc) -create schema "idz"; - -create type "idz_gender" as enum ('male', 'female'); - create table "idz_profile" ( - "id" bigint primary key not null, - "player_id" bigint not null + "id" integer primary key not null, + "player_id" integer not null references "aime_player"("id") on delete cascade, -- TODO shop_id "name" text not null, - "lv" smallint not null, + "lv" integer not null, "exp" integer not null, "fame" integer not null, "dpoint" integer not null, @@ -24,10 +20,10 @@ create table "idz_profile" ( ); create table "idz_chara" ( - "id" bigint primary key not null + "id" integer primary key not null references "idz_profile"("id") on delete cascade, - "gender" "idz_gender" not null, + "gender" text not null, "field_02" integer not null, "field_04" integer not null, "field_06" integer not null, @@ -40,18 +36,18 @@ create table "idz_chara" ( ); create table "idz_course_plays" ( - "id" bigint primary key not null, - "profile_id" bigint not null + "id" integer primary key not null, + "profile_id" integer not null references "idz_profile"("id") on delete cascade, - "course_no" smallint not null, + "course_no" integer not null, "count" integer not null, constraint "idz_course_plays_uq" unique ("profile_id", "course_no") ); create table "idz_car" ( - "id" bigint primary key not null, - "profile_id" bigint not null + "id" integer primary key not null, + "profile_id" integer not null references "idz_profile"("id") on delete cascade, "selector" integer not null, @@ -73,22 +69,22 @@ create table "idz_car" ( ); create table "idz_car_selection" ( - "id" bigint primary key not null + "id" integer primary key not null references "idz_profile"("id") on delete cascade, - "car_id" bigint not null + "car_id" integer not null references "idz_car"("id") on delete cascade ); create table "idz_solo_mission_state" ( - "id" bigint primary key not null, - "profile_id" bigint not null + "id" integer primary key not null, + "profile_id" integer not null references "idz_profile"("id") on delete cascade, - "grid_no" smallint not null, - "cell_no" smallint not null, - "value" smallint not null, + "grid_no" integer not null, + "cell_no" integer not null, + "value" integer not null, constraint "idz_solo_mission_state_uq" unique ( "profile_id", "grid_no", @@ -97,32 +93,32 @@ create table "idz_solo_mission_state" ( ); create table "idz_settings" ( - "id" bigint primary key not null + "id" integer primary key not null references "idz_profile"("id") on delete cascade, - "music" smallint not null, - "pack" bigint not null, - "paper_cup" smallint not null, -- Not a boolean, oddly enough + "music" integer not null, + "pack" integer not null, + "paper_cup" integer not null, -- Not a boolean, oddly enough "gauges" integer not null ); create table "idz_story_state" ( - "id" bigint primary key not null + "id" integer primary key not null references "idz_profile"("id") on delete cascade, - "x" smallint not null, - "y" smallint not null + "x" integer not null, + "y" integer not null ); create table "idz_story_cell_state" ( - "id" bigint primary key not null, - "profile_id" bigint not null + "id" integer primary key not null, + "profile_id" integer not null references "idz_profile"("id") on delete cascade, "row_no" integer not null, "col_no" integer not null, - "a" smallint not null, - "b" smallint not null, + "a" integer not null, + "b" integer not null, constraint "idz_story_cell_state_uq" unique ( "profile_id", "row_no", @@ -131,7 +127,7 @@ create table "idz_story_cell_state" ( ); create table "idz_free_car" ( - "id" bigint primary key not null + "id" integer primary key not null references "idz_profile"("id") on delete cascade, "valid_from" timestamp not null @@ -141,29 +137,29 @@ create table "idz_free_car" ( -- Times are stored as floating-point seconds create table "idz_ta_result" ( - "id" bigint primary key not null, - "profile_id" bigint not null + "id" integer primary key not null, + "profile_id" integer not null references "idz_profile"("id") on delete cascade, - "route_no" smallint not null, + "route_no" integer not null, "total_time" float not null, "section_times" text not null, - "flags" smallint not null, - "grade" smallint not null, + "flags" integer not null, + "grade" integer not null, "car_selector" integer not null, "timestamp" timestamp not null ); create table "idz_ta_best" ( - "id" bigint primary key not null, - "profile_id" bigint not null + "id" integer primary key not null, + "profile_id" integer not null references "idz_profile"("id") on delete cascade, - "route_no" smallint not null, + "route_no" integer not null, "total_time" float not null, "section_times" text not null, - "flags" smallint not null, - "grade" smallint not null, -- TODO enum + "flags" integer not null, + "grade" integer not null, -- TODO enum "car_selector" integer not null, "timestamp" timestamp not null, constraint "idz_ta_best_uq" unique ("profile_id", "route_no") @@ -172,11 +168,11 @@ create table "idz_ta_best" ( -- TODO ta_section_best create table "idz_background_unlock" ( - "id" bigint primary key not null, - "profile_id" bigint not null + "id" integer primary key not null, + "profile_id" integer not null references "idz_profile"("id") on delete cascade, - "background_no" smallint not null, + "background_no" integer not null, constraint "idz_background_unlock_uq" unique ( "profile_id", "background_no" @@ -184,48 +180,48 @@ create table "idz_background_unlock" ( ); create table "idz_title_unlock" ( - "id" bigint primary key not null, - "profile_id" bigint not null + "id" integer primary key not null, + "profile_id" integer not null references "idz_profile"("id") on delete cascade, - "title_no" smallint not null, + "title_no" integer not null, constraint "idz_title_unlock_uq" unique ("profile_id", "title_no") ); create table "idz_unlocks" ( - "id" bigint primary key not null + "id" integer primary key not null references "idz_profile"("id") on delete cascade, - "cup" smallint not null, + "cup" integer not null, "gauges" integer not null, "music" integer not null, "last_mileage_reward" integer not null ); create table "idz_team" ( - "id" bigint primary key not null, + "id" integer primary key not null, "ext_id" integer not null, "name" text not null, - "name_bg" smallint not null, - "name_fx" smallint not null, + "name_bg" integer not null, + "name_fx" integer not null, "register_time" timestamp not null, constraint "idz_team_uq" unique ("ext_id") ); create table "idz_team_auto" ( - "id" bigint primary key not null + "id" integer primary key not null references "idz_team"("id") on delete cascade, - "serial_no" smallint not null, - "name_idx" smallint not null, + "serial_no" integer not null, + "name_idx" integer not null, constraint "idz_team_auto_uq" unique ("serial_no", "name_idx") ); create table "idz_team_member" ( - "id" bigint primary key not null + "id" integer primary key not null references "idz_profile"("id") on delete cascade, - "team_id" bigint not null + "team_id" integer not null references "idz_team"("id") on delete cascade, "join_time" timestamp not null, @@ -233,10 +229,10 @@ create table "idz_team_member" ( ); create table "idz_team_reservation" ( - "id" bigint primary key not null + "id" integer primary key not null references "aime_player"("id") on delete cascade, - "team_id" bigint not null + "team_id" integer not null references "idz_team"("id") on delete cascade, "join_time" timestamp not null, diff --git a/schema/migrate/M0001-add-helper-funcs.sql b/schema/migrate/M0001-add-helper-funcs.sql deleted file mode 100644 index 7302f4b..0000000 --- a/schema/migrate/M0001-add-helper-funcs.sql +++ /dev/null @@ -1,81 +0,0 @@ --- Helper functions which might be useful in one-off queries. --- These are not utilised by any services; those have their own ID generators. - -create or replace function idnum( - str text) - returns bigint as $$ -declare - num bigint; - x integer; -begin - num := 0; - - for i in 1..11 loop - x := ascii(substr(str, i, 1)); - num := num << 6; - - if x >= 65 and x <= 90 then - -- A to Z - num = num | (x - 65); - elsif x >= 97 and x <= 122 then - -- a to z - num = num | (x - 71); - elsif x >= 48 and x <= 58 then - -- 0 to 9 - num = num | (x + 4); - elsif x = 45 then - -- Dash - num = num | 62; - elsif x = 95 then - -- Underscore - num = num | 63; - else - raise 'Bad input'; - end if; - end loop; - - return num; -end; -$$ language plpgsql; - -create or replace function idstr( - num bigint) - returns text as $$ -declare - a text; - pos integer; - str text; -begin - a := 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'; - str := ''; - - for i in 1..11 loop - pos := (num & 63); - str := substr(a, pos + 1, 1) || str; - num := num >> 6; - end loop; - - return str; -end; -$$ language plpgsql; - -create or replace function newid() - returns bigint as $$ -declare - bytes bytea; - result bigint; -begin - bytes := gen_random_bytes(8); - result := 0; - - for i in 0..7 loop - result := (result << 8) | get_byte(bytes, i); - end loop; - - -- Truncate high bit to ensure result is positive - -- This value is just the decimal representation of 0x7FFFFFFF`FFFFFFFF. - - return result & 9223372036854775807; -end; -$$ language plpgsql; - diff --git a/schema/migrate/M0002-idz-drop-profile-ext-id.sql b/schema/migrate/M0002-idz-drop-profile-ext-id.sql deleted file mode 100644 index ad9780a..0000000 --- a/schema/migrate/M0002-idz-drop-profile-ext-id.sql +++ /dev/null @@ -1,3 +0,0 @@ -alter table "idz"."profile" drop column "ext_id"; - -update "meta" set "schemaver" = 1; diff --git a/schema/migrate/M0003-idz-add-team.sql b/schema/migrate/M0003-idz-add-team.sql deleted file mode 100644 index 8b49b3b..0000000 --- a/schema/migrate/M0003-idz-add-team.sql +++ /dev/null @@ -1,42 +0,0 @@ -create table "idz"."team" ( - "id" bigint primary key not null, - "ext_id" integer not null, - "name" text not null, - "name_bg" smallint not null, - "name_fx" smallint not null, - "register_time" timestamp not null, - constraint "team_uq" unique ("ext_id") -); - -create table "idz"."team_auto" ( - "id" bigint primary key not null - references "idz"."team"("id") - on delete cascade, - "serial_no" smallint not null, - "name_idx" smallint not null, - constraint "team_auto_uq" unique ("serial_no", "name_idx") -); - -create table "idz"."team_member" ( - "id" bigint primary key not null - references "idz"."profile"("id") - on delete cascade, - "team_id" bigint not null - references "idz"."team"("id") - on delete cascade, - "join_time" timestamp not null, - "leader" boolean not null -); - -create table "idz"."team_reservation" ( - "id" bigint primary key not null - references "aime"."player"("id") - on delete cascade, - "team_id" bigint not null - references "idz"."team"("id") - on delete cascade, - "join_time" timestamp not null, - "leader" boolean not null -); - -update "meta" set "schemaver" = 2; diff --git a/schema/migrate/M0004-idz-fix-col-casing.sql b/schema/migrate/M0004-idz-fix-col-casing.sql deleted file mode 100644 index 2e8f894..0000000 --- a/schema/migrate/M0004-idz-fix-col-casing.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Fun story: PostgreSQL (can't speak for other dbs) doesn't actually treat --- its column and table names case-insensitively. It forces them to lower- --- case instead. If you quote an upper-case field name then it will be upper- --- case in the database and you will need to use quoted upper-case every time --- you want to access it. - --- It is likely that people will find this behavior surprising, to say the --- least. We'll change the column names so that they are always lower-case. - -alter table "idz"."chara" rename column "field_0A" to "field_0a"; -alter table "idz"."chara" rename column "field_0C" to "field_0c"; -alter table "idz"."chara" rename column "field_0E" to "field_0e"; - -alter table "idz"."car" rename column "field_4A" to "field_4a"; -alter table "idz"."car" rename column "field_4C" to "field_4c"; -alter table "idz"."car" rename column "field_5A" to "field_5a"; -alter table "idz"."car" rename column "field_5B" to "field_5b"; -alter table "idz"."car" rename column "field_5C" to "field_5c"; -alter table "idz"."car" rename column "field_5E" to "field_5e"; - -update "meta" set "schemaver" = 3; diff --git a/schema/migrate/M0005-merge-schemas.sql b/schema/migrate/M0005-merge-schemas.sql deleted file mode 100644 index b88bc6d..0000000 --- a/schema/migrate/M0005-merge-schemas.sql +++ /dev/null @@ -1,81 +0,0 @@ -alter type "aime"."region" set schema "public"; -alter type "region" rename to "aime_region"; - -alter table "aime"."shop" set schema "public"; -alter table "shop" rename to "aime_shop"; - -alter table "aime"."machine" set schema "public"; -alter table "machine" rename to "aime_machine"; - -alter table "aime"."player" set schema "public"; -alter table "player" rename to "aime_player"; - -alter table "aime"."card" set schema "public"; -alter table "card" rename to "aime_card"; - ----- - -alter type "idz"."gender" set schema "public"; -alter type "gender" rename to "idz_gender"; - -alter table "idz"."profile" set schema "public"; -alter table "profile" rename to "idz_profile"; - -alter table "idz"."chara" set schema "public"; -alter table "chara" rename to "idz_chara"; - -alter table "idz"."course_plays" set schema "public"; -alter table "course_plays" rename to "idz_course_plays"; - -alter table "idz"."car" set schema "public"; -alter table "car" rename to "idz_car"; - -alter table "idz"."car_selection" set schema "public"; -alter table "car_selection" rename to "idz_car_selection"; - -alter table "idz"."solo_mission_state" set schema "public"; -alter table "solo_mission_state" rename to "idz_solo_mission_state"; - -alter table "idz"."settings" set schema "public"; -alter table "settings" rename to "idz_settings"; - -alter table "idz"."story_state" set schema "public"; -alter table "story_state" rename to "idz_story_state"; - -alter table "idz"."story_cell_state" set schema "public"; -alter table "story_cell_state" rename to "idz_story_cell_state"; - -alter table "idz"."free_car" set schema "public"; -alter table "free_car" rename to "idz_free_car"; - -alter table "idz"."ta_result" set schema "public"; -alter table "ta_result" rename to "idz_ta_result"; - -alter table "idz"."ta_best" set schema "public"; -alter table "ta_best" rename to "idz_ta_best"; - -alter table "idz"."background_unlock" set schema "public"; -alter table "background_unlock" rename to "idz_background_unlock"; - -alter table "idz"."title_unlock" set schema "public"; -alter table "title_unlock" rename to "idz_title_unlock"; - -alter table "idz"."unlocks" set schema "public"; -alter table "unlocks" rename to "idz_unlocks"; - -alter table "idz"."team" set schema "public"; -alter table "team" rename to "idz_team"; - -alter table "idz"."team_auto" set schema "public"; -alter table "team_auto" rename to "idz_team_auto"; - -alter table "idz"."team_member" set schema "public"; -alter table "team_member" rename to "idz_team_member"; - -alter table "idz"."team_reservation" set schema "public"; -alter table "team_reservation" rename to "idz_team_reservation"; - -drop schema "idz"; -drop schema "aime"; - -update "meta" set "schemaver" = 5; diff --git a/schema/migrate/M0006-flatten-arrays.sql b/schema/migrate/M0006-flatten-arrays.sql deleted file mode 100644 index a4d3973..0000000 --- a/schema/migrate/M0006-flatten-arrays.sql +++ /dev/null @@ -1,19 +0,0 @@ -alter table "idz_car" rename column "field_04" to "tmp"; -alter table "idz_car" add column "field_04" text; -update "idz_car" set "field_04" = array_to_string("tmp", ','); -alter table "idz_car" alter column "field_04" set not null; -alter table "idz_car" drop column "tmp"; - -alter table "idz_ta_result" rename column "section_times" to "tmp"; -alter table "idz_ta_result" add column "section_times" text; -update "idz_ta_result" set "section_times" = array_to_string("tmp", ','); -alter table "idz_ta_result" alter column "section_times" set not null; -alter table "idz_ta_result" drop column "tmp"; - -alter table "idz_ta_best" rename column "section_times" to "tmp"; -alter table "idz_ta_best" add column "section_times" text; -update "idz_ta_best" set "section_times" = array_to_string("tmp", ','); -alter table "idz_ta_best" alter column "section_times" set not null; -alter table "idz_ta_best" drop column "tmp"; - -update "meta" set "schemaver" = 6; diff --git a/src/index.ts b/src/index.ts index 591ee14..853a9c7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,10 +12,12 @@ import chunithm from "./chunithm"; import diva from "./diva"; import idz from "./idz"; import idzPing from "./idz/ping"; -import { openDataSource } from "./sql"; +import { openSqlite } from "./sql"; import * as Swb from "./switchboard"; -const db = openDataSource(); +fs.mkdirSync("./data", { recursive: true }); + +const db = openSqlite("./data/db"); const tls = { cert: fs.readFileSync("pki/server.pem"), diff --git a/src/sql/index.ts b/src/sql/index.ts index fc5c101..5beb3b9 100644 --- a/src/sql/index.ts +++ b/src/sql/index.ts @@ -1,3 +1,3 @@ export * from "./api"; -export * from "./pg"; +export * from "./sqlite"; export * from "./util"; diff --git a/src/sql/pg.ts b/src/sql/pg.ts deleted file mode 100644 index 1e56c25..0000000 --- a/src/sql/pg.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Pool, PoolClient } from "pg"; -import * as sql from "sql-bricks-postgres"; - -import { DataSource, Row, Transaction } from "./api"; - -class PgTransaction implements Transaction { - constructor(private readonly _conn: PoolClient) {} - - async modify(stmt: sql.Statement): Promise { - await this._conn.query(stmt.toParams()); - } - - async fetchRow(stmt: sql.SelectStatement): Promise { - const { rows } = await this._conn.query(stmt.toParams()); - - return rows[0]; - } - - async fetchRows(stmt: sql.SelectStatement): Promise { - const { rows } = await this._conn.query(stmt.toParams()); - - return rows; - } -} - -class PgDataSource implements DataSource { - private readonly _pool: Pool; - - constructor() { - this._pool = new Pool(); - } - - async transaction( - callback: (txn: Transaction) => Promise - ): Promise { - const conn = await this._pool.connect(); - - await conn.query("begin"); - - try { - const txn = new PgTransaction(conn); - const result = await callback(txn); - - await conn.query("commit"); - - return result; - } catch (e) { - await conn.query("rollback"); - - return Promise.reject(e); - } - } -} - -export function openDataSource(): DataSource { - return new PgDataSource(); -} diff --git a/src/sql/sqlite.ts b/src/sql/sqlite.ts new file mode 100644 index 0000000..240237d --- /dev/null +++ b/src/sql/sqlite.ts @@ -0,0 +1,104 @@ +import Database from "better-sqlite3"; +import * as sql from "sql-bricks-postgres"; + +import { DataSource, Row, Transaction } from "./api"; + +// bless me father for i have sinned +const fuFixup = new RegExp(" FOR UPDATE$"); + +function _preprocess(stmt: sql.Statement) { + const params = stmt.toParams({ placeholder: "?" }); + const values = new Array(); + + for (const value of params.values) { + // Pass null through as-is + // Pass dates as ISO strings in UTC + // Pass everything else (numbers, booleans, BigInts) as their string rep. + + if (value === null) { + values.push(null); + } else if (value instanceof Date) { + values.push(value.toISOString()); + } else { + values.push(value.toString()); + } + } + + // Use string manipulation to cut off any trailing " FOR UPDATE" clause + // in a SELECT statement, since SQLite doesn't support it. I really hope + // that performing string manipulation on SQL code like this doesn't come + // back to bite me. + + return { + text: params.text.replace(fuFixup, ""), + values, + }; +} + +function _postprocess(obj: {}): Row { + const result = {}; + + for (const [k, v] of Object.entries(obj)) { + result[k] = v.toString(); + } + + return result; +} + +class SqliteTransaction implements Transaction { + constructor(private readonly _db: Database.Database) {} + + modify(stmt: sql.Statement): Promise { + const params = _preprocess(stmt); + + this._db.prepare(params.text).run(...params.values); + + return Promise.resolve(); + } + + fetchRow(stmt: sql.SelectStatement): Promise { + const params = _preprocess(stmt); + const raw = this._db.prepare(params.text).get(...params.values); + const result = raw && _postprocess(raw); + + return Promise.resolve(result); + } + + fetchRows(stmt: sql.SelectStatement): Promise { + const params = _preprocess(stmt); + const raw = this._db.prepare(params.text).all(...params.values); + const result = raw.map(_postprocess); + + return Promise.resolve(result); + } +} + +class SqliteDataSource implements DataSource { + constructor(private readonly _path: string) {} + + async transaction( + callback: (txn: Transaction) => Promise + ): Promise { + const db = new Database(this._path); + + db.defaultSafeIntegers(); + db.prepare("begin").run(); + + try { + const txn = new SqliteTransaction(db); + const result = await callback(txn); + + db.prepare("commit").run(); + + return result; + } catch (e) { + db.prepare("rollback").run(); + + return Promise.reject(e); + } + } +} + +export function openSqlite(path: string): DataSource { + return new SqliteDataSource(path); +} diff --git a/yarn.lock b/yarn.lock index 4037e85..31070f9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -309,6 +309,13 @@ dependencies: "@babel/types" "^7.3.0" +"@types/better-sqlite3@^5.4.0": + version "5.4.0" + resolved "https://registry.yarnpkg.com/@types/better-sqlite3/-/better-sqlite3-5.4.0.tgz#ab7336ccc2e31bd88247016c675cfcb01df99787" + integrity sha512-nzm7lJ7l3jBmGUbtkL8cdOMhPkN6Pw2IM+b0V7iIKba+YKiLrjkIy7vVLsBIVnd7+lgzBzrHsXZxCaFTcmw5Ow== + dependencies: + "@types/integer" "*" + "@types/body-parser@*": version "1.17.0" resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.17.0.tgz#9f5c9d9bd04bb54be32d5eb9fc0d8c974e6cf58c" @@ -360,6 +367,11 @@ "@types/express-serve-static-core" "*" "@types/serve-static" "*" +"@types/integer@*": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@types/integer/-/integer-1.0.0.tgz#f5b313876012fad0afeb5318f03cb871064eb33e" + integrity sha512-3viiRKLoSP2Qr78nMoQjkDc0fan4BgmpOyV1+1gKjE8wWXo3QQ78WItO6f9WuBf3qe3ymDYhM65oqHTOZ0rFxw== + "@types/istanbul-lib-coverage@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-1.1.0.tgz#2cc2ca41051498382b43157c8227fea60363f94a" @@ -394,21 +406,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-11.11.3.tgz#7c6b0f8eaf16ae530795de2ad1b85d34bf2f5c58" integrity sha512-wp6IOGu1lxsfnrD+5mX6qwSwWuqsdkKKxTN4aQc4wByHAKZJf9/D4KXPQ1POUjEbnCP5LMggB0OEFNY9OTsMqg== -"@types/pg-types@*": - version "1.11.4" - resolved "https://registry.yarnpkg.com/@types/pg-types/-/pg-types-1.11.4.tgz#8d7c59fb509ce3dca3f8bae589252051c639a9a8" - integrity sha512-WdIiQmE347LGc1Vq3Ki8sk3iyCuLgnccqVzgxek6gEHp2H0p3MQ3jniIHt+bRODXKju4kNQ+mp53lmP5+/9moQ== - dependencies: - moment ">=2.14.0" - -"@types/pg@^7.4.14": - version "7.4.14" - resolved "https://registry.yarnpkg.com/@types/pg/-/pg-7.4.14.tgz#8235910120e81ca671e0e62b7de77d048b9a22b2" - integrity sha512-2e4XapP9V/X42IGByC5IHzCzHqLLCNJid8iZBbkk6lkaDMvli8Rk62YE9wjGcLD5Qr5Zaw1ShkQyXy91PI8C0Q== - dependencies: - "@types/node" "*" - "@types/pg-types" "*" - "@types/range-parser@*": version "1.2.3" resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.3.tgz#7ee330ba7caafb98090bece86a5ee44115904c2c" @@ -697,6 +694,14 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" +better-sqlite3@^5.4.3: + version "5.4.3" + resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-5.4.3.tgz#2cb843ce14c56de9e9c0ca6b89844b7d4b5794b8" + integrity sha512-fPp+8f363qQIhuhLyjI4bu657J/FfMtgiiHKfaTsj3RWDkHlWC1yT7c6kHZDnBxzQVoAINuzg553qKmZ4F1rEw== + dependencies: + integer "^2.1.0" + tar "^4.4.10" + body-parser@1.18.3: version "1.18.3" resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.18.3.tgz#5b292198ffdd553b3a0f20ded0592b956955c8b4" @@ -768,11 +773,6 @@ buffer-from@1.x, buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== -buffer-writer@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-2.0.0.tgz#ce7eb81a38f7829db09c873f2fbb792c0c98ec04" - integrity sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw== - builtin-modules@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" @@ -843,6 +843,11 @@ chalk@^2.4.2: escape-string-regexp "^1.0.5" supports-color "^5.3.0" +chownr@^1.1.1: + version "1.1.3" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.3.tgz#42d837d5239688d55f303003a508230fa6727142" + integrity sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw== + ci-info@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" @@ -1462,6 +1467,13 @@ fresh@0.5.2: resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac= +fs-minipass@^1.2.5: + version "1.2.7" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-1.2.7.tgz#ccff8570841e7fe4265693da88936c55aed7f7c7" + integrity sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA== + dependencies: + minipass "^2.6.0" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -1682,6 +1694,11 @@ inherits@2, inherits@2.0.3, inherits@~2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= +integer@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/integer/-/integer-2.1.0.tgz#29134ea2f7ba3362ed4dbe6bcca992b1f18ff276" + integrity sha512-vBtiSgrEiNocWvvZX1RVfeOKa2mCHLZQ2p9nkQkQZ/BvEiY+6CcUz0eyjvIiewjJoeNidzg2I+tpPJvpyspL1w== + invariant@^2.2.4: version "2.2.4" resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6" @@ -2629,6 +2646,21 @@ minimist@~0.0.1: resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" integrity sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8= +minipass@^2.6.0, minipass@^2.8.6, minipass@^2.9.0: + version "2.9.0" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.9.0.tgz#e713762e7d3e32fed803115cf93e04bca9fcc9a6" + integrity sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg== + dependencies: + safe-buffer "^5.1.2" + yallist "^3.0.0" + +minizlib@^1.2.1: + version "1.3.3" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.3.3.tgz#2290de96818a34c29551c8a8d301216bd65a861d" + integrity sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q== + dependencies: + minipass "^2.9.0" + mixin-deep@^1.2.0: version "1.3.1" resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.1.tgz#a49e7268dce1a0d9698e45326c5626df3543d0fe" @@ -2637,18 +2669,13 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -mkdirp@0.x, mkdirp@^0.5.1: +mkdirp@0.x, mkdirp@^0.5.0, mkdirp@^0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" integrity sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM= dependencies: minimist "0.0.8" -moment@>=2.14.0: - version "2.24.0" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b" - integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg== - morgan@^1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/morgan/-/morgan-1.9.1.tgz#0a8d16734a1d9afbc824b99df87e738e58e2da59" @@ -2901,11 +2928,6 @@ p-try@^2.0.0: resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.0.0.tgz#85080bb87c64688fa47996fe8f7dfbe8211760b1" integrity sha512-hMp0onDKIajHfIkdRk3P4CdCmErkYAxxDtP3Wx/4nZ3aGlau2VKh3mZpcuFkH27WQkL/3WBCPOktzA9ZOAnMQQ== -packet-reader@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74" - integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ== - parse-json@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-4.0.0.tgz#be35f5425be1f7f6c747184f98a788cb99477ee0" @@ -2971,52 +2993,6 @@ performance-now@^2.1.0: resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= -pg-connection-string@0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" - integrity sha1-2hhHsglA5C7hSSvq9l1J2RskXfc= - -pg-int8@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" - integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== - -pg-pool@^2.0.4: - version "2.0.6" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-2.0.6.tgz#7b561a482feb0a0e599b58b5137fd2db3ad8111c" - integrity sha512-hod2zYQxM8Gt482q+qONGTYcg/qVcV32VHVPtktbBJs0us3Dj7xibISw0BAAXVMCzt8A/jhfJvpZaxUlqtqs0g== - -pg-types@~2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.0.1.tgz#b8585a37f2a9c7b386747e44574799549e5f4933" - integrity sha512-b7y6QM1VF5nOeX9ukMQ0h8a9z89mojrBHXfJeSug4mhL0YpxNBm83ot2TROyoAmX/ZOX3UbwVO4EbH7i1ZZNiw== - dependencies: - pg-int8 "1.0.1" - postgres-array "~2.0.0" - postgres-bytea "~1.0.0" - postgres-date "~1.0.4" - postgres-interval "^1.1.0" - -pg@^7.10.0: - version "7.10.0" - resolved "https://registry.yarnpkg.com/pg/-/pg-7.10.0.tgz#2a359ee29ed1971344ac7f44317a9d1bcd80a8ff" - integrity sha512-aE6FZomsyn3OeGv1oM50v7Xu5zR75c15LXdOCwA9GGrfjXsQjzwYpbcTS6OwEMhYfZQS6m/FVU/ilPLiPzJDCw== - dependencies: - buffer-writer "2.0.0" - packet-reader "1.0.0" - pg-connection-string "0.1.3" - pg-pool "^2.0.4" - pg-types "~2.0.0" - pgpass "1.x" - semver "4.3.2" - -pgpass@1.x: - version "1.0.2" - resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" - integrity sha1-Knu0G2BltnkH6R2hsHwYR8h3swY= - dependencies: - split "^1.0.0" - pify@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176" @@ -3046,28 +3022,6 @@ posix-character-classes@^0.1.0: resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab" integrity sha1-AerA/jta9xoqbAL+q7jB/vfgDqs= -postgres-array@~2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e" - integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA== - -postgres-bytea@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35" - integrity sha1-AntTPAqokOJtFy1Hz5zOzFIazTU= - -postgres-date@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.4.tgz#1c2728d62ef1bff49abdd35c1f86d4bdf118a728" - integrity sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA== - -postgres-interval@^1.1.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695" - integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ== - dependencies: - xtend "^4.0.0" - prelude-ls@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" @@ -3369,11 +3323,6 @@ sax@^1.2.4: resolved "https://registry.yarnpkg.com/semver/-/semver-5.6.0.tgz#7e74256fbaa49c75aa7c7a205cc22799cac80004" integrity sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg== -semver@4.3.2: - version "4.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" - integrity sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c= - send@0.16.2: version "0.16.2" resolved "https://registry.yarnpkg.com/send/-/send-0.16.2.tgz#6ecca1e0f8c156d141597559848df64730a6bbc1" @@ -3567,13 +3516,6 @@ split-string@^3.0.1, split-string@^3.0.2: dependencies: extend-shallow "^3.0.0" -split@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" - integrity sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg== - dependencies: - through "2" - sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" @@ -3726,6 +3668,19 @@ symbol-tree@^3.2.2: resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.2.tgz#ae27db38f660a7ae2e1c3b7d1bc290819b8519e6" integrity sha1-rifbOPZgp64uHDt9G8KQgZuFGeY= +tar@^4.4.10: + version "4.4.13" + resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.13.tgz#43b364bc52888d555298637b10d60790254ab525" + integrity sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA== + dependencies: + chownr "^1.1.1" + fs-minipass "^1.2.5" + minipass "^2.8.6" + minizlib "^1.2.1" + mkdirp "^0.5.0" + safe-buffer "^5.1.2" + yallist "^3.0.3" + test-exclude@^5.0.0: version "5.1.0" resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-5.1.0.tgz#6ba6b25179d2d38724824661323b73e03c0c1de1" @@ -3741,11 +3696,6 @@ throat@^4.0.0: resolved "https://registry.yarnpkg.com/throat/-/throat-4.1.0.tgz#89037cbc92c56ab18926e6ba4cbb200e15672a6a" integrity sha1-iQN8vJLFarGJJua6TLsgDhVnKmo= -through@2: - version "2.3.8" - resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= - tmpl@1.0.x: version "1.0.4" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1" @@ -4084,16 +4034,16 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== -xtend@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af" - integrity sha1-pcbVMr5lbiPbgg77lDofBJmNY68= - "y18n@^3.2.1 || ^4.0.0": version "4.0.0" resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b" integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w== +yallist@^3.0.0, yallist@^3.0.3: + version "3.1.1" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" + integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== + yargs-parser@10.x: version "10.1.0" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-10.1.0.tgz#7202265b89f7e9e9f2e5765e0fe735a905edbaa8"