/** * Pokemon Showdown log viewer * * by Zarel * @license MIT */ import {FS} from "../../lib/fs"; import {Utils} from '../../lib/utils'; import * as Dashycode from '../../lib/dashycode'; import {QueryProcessManager, exec} from "../../lib/process-manager"; import {Repl} from '../../lib/repl'; import {Config} from '../config-loader'; import {Dex} from '../../sim/dex'; import {Chat} from '../chat'; const DAY = 24 * 60 * 60 * 1000; const MAX_RESULTS = 3000; const MAX_MEMORY = 67108864; // 64MB const MAX_PROCESSES = 1; const MAX_TOPUSERS = 100; const CHATLOG_PM_TIMEOUT = 1 * 60 * 60 * 1000; // 1 hour const UPPER_STAFF_ROOMS = ['upperstaff', 'adminlog']; interface ChatlogSearch { raw?: boolean; search: string; room: RoomID; date: string; limit?: number | null; args?: string[]; } export class LogReaderRoom { roomid: RoomID; constructor(roomid: RoomID) { this.roomid = roomid; } async listMonths() { try { const listing = await FS(`logs/chat/${this.roomid}`).readdir(); return listing.filter(file => /^[0-9][0-9][0-9][0-9]-[0-9][0-9]$/.test(file)); } catch (err) { return []; } } async listDays(month: string) { try { const listing = await FS(`logs/chat/${this.roomid}/${month}`).readdir(); return listing.filter(file => file.endsWith(".txt")).map(file => file.slice(0, -4)); } catch (err) { return []; } } async getLog(day: string) { const month = LogReader.getMonth(day); const log = FS(`logs/chat/${this.roomid}/${month}/${day}.txt`); if (!await log.exists()) return null; return log.createReadStream(); } } export const LogReader = new class { async get(roomid: RoomID) { if (!await FS(`logs/chat/${roomid}`).exists()) return null; return new LogReaderRoom(roomid); } async list() { const listing = await FS(`logs/chat`).readdir(); return listing.filter(file => /^[a-z0-9-]+$/.test(file)) as RoomID[]; } async listCategorized(user: User, opts?: string) { const list = await this.list(); const isUpperStaff = user.can('rangeban'); const isStaff = user.can('lock'); const official = []; const normal = []; const hidden = []; const secret = []; const deleted = []; const personal: RoomID[] = []; const deletedPersonal: RoomID[] = []; let atLeastOne = false; for (const roomid of list) { const room = Rooms.get(roomid); const forceShow = room && ( // you are authed in the room (room.auth.has(user.id) && user.can('mute', null, room)) || // you are staff and currently in the room (isStaff && user.inRooms.has(room.roomid)) ); if (!isUpperStaff && !forceShow) { if (!isStaff) continue; if (!room) continue; if (!room.checkModjoin(user)) continue; if (room.settings.isPrivate === true) continue; } atLeastOne = true; if (roomid.includes('-')) { const matchesOpts = opts && roomid.startsWith(`${opts}-`); if (matchesOpts || opts === 'all' || forceShow) { (room ? personal : deletedPersonal).push(roomid); } } else if (!room) { if (opts === 'all' || opts === 'deleted') deleted.push(roomid); } else if (room.settings.isOfficial) { official.push(roomid); } else if (!room.settings.isPrivate) { normal.push(roomid); } else if (room.settings.isPrivate === 'hidden') { hidden.push(roomid); } else { secret.push(roomid); } } if (!atLeastOne) return null; return {official, normal, hidden, secret, deleted, personal, deletedPersonal}; } async read(roomid: RoomID, day: string, limit: number) { const roomLog = await LogReader.get(roomid); const stream = await roomLog!.getLog(day); let buf = ''; let i = (LogSearcher as FSLogSearcher).results || 0; if (!stream) { buf += `

Room "${roomid}" doesn't have logs for ${day}

`; } else { for await (const line of stream.byLine()) { const rendered = LogViewer.renderLine(line); if (rendered) { buf += `${line}\n`; i++; if (i > limit) break; } } } return buf; } getMonth(day?: string) { if (!day) day = Chat.toTimestamp(new Date()).split(' ')[0]; return day.slice(0, 7); } nextDay(day: string) { const nextDay = new Date(new Date(day).getTime() + DAY); return nextDay.toISOString().slice(0, 10); } prevDay(day: string) { const prevDay = new Date(new Date(day).getTime() - DAY); return prevDay.toISOString().slice(0, 10); } nextMonth(month: string) { const nextMonth = new Date(new Date(`${month}-15`).getTime() + 30 * DAY); return nextMonth.toISOString().slice(0, 7); } prevMonth(month: string) { const prevMonth = new Date(new Date(`${month}-15`).getTime() - 30 * DAY); return prevMonth.toISOString().slice(0, 7); } today() { return Chat.toTimestamp(new Date()).slice(0, 10); } isMonth(text: string) { return /[0-9]{4}-[0-9]{2}/.test(text); } isDay(text: string) { return /[0-9]{4}-[0-9]{2}-[0-9]{2}/.test(text); } async findBattleLog(tier: ID, number: number): Promise { // binary search! const months = (await FS('logs').readdir()).filter(this.isMonth).sort(); if (!months.length) return null; // find first day let firstDay!: string; while (months.length) { const month = months[0]; try { const days = (await FS(`logs/${month}/${tier}/`).readdir()).filter(this.isDay).sort(); firstDay = days[0]; break; } catch (err) {} months.shift(); } if (!firstDay) return null; // find last day let lastDay!: string; while (months.length) { const month = months[months.length - 1]; try { const days = (await FS(`logs/${month}/${tier}/`).readdir()).filter(this.isDay).sort(); lastDay = days[days.length - 1]; break; } catch (err) {} months.pop(); } if (!lastDay) throw new Error(`getBattleLog month range search for ${tier}`); const getBattleNum = (battleName: string) => Number(battleName.split('-')[1].slice(0, -9)); const getDayRange = async (day: string) => { const month = day.slice(0, 7); try { const battles = (await FS(`logs/${month}/${tier}/${day}`).readdir()).filter( b => b.endsWith('.log.json') ); Utils.sortBy(battles, getBattleNum); return [getBattleNum(battles[0]), getBattleNum(battles[battles.length - 1])]; } catch (err) { return null; } }; const dayExists = (day: string) => FS(`logs/${day.slice(0, 7)}/${tier}/${day}`).exists(); const nextExistingDay = async (day: string) => { for (let i = 0; i < 3650; i++) { day = this.nextDay(day); if (await dayExists(day)) return day; if (day === lastDay) return null; } return null; }; const prevExistingDay = async (day: string) => { for (let i = 0; i < 3650; i++) { day = this.prevDay(day); if (await dayExists(day)) return day; if (day === firstDay) return null; } return null; }; for (let i = 0; i < 100; i++) { const middleDay = new Date( (new Date(firstDay).getTime() + new Date(lastDay).getTime()) / 2 ).toISOString().slice(0, 10); let currentDay: string | null = middleDay; let dayRange = await getDayRange(middleDay); if (!dayRange) { currentDay = await nextExistingDay(middleDay); if (!currentDay) { const lastExistingDay = await prevExistingDay(middleDay); if (!lastExistingDay) throw new Error(`couldn't find existing day`); lastDay = lastExistingDay; continue; } dayRange = await getDayRange(currentDay); if (!dayRange) throw new Error(`existing day was a lie`); } const [lowest, highest] = dayRange; if (number < lowest) { // before currentDay if (firstDay === currentDay) return null; lastDay = this.prevDay(currentDay); } else if (number > highest) { // after currentDay if (lastDay === currentDay) return null; firstDay = this.nextDay(currentDay); } else { // during currentDay const month = currentDay.slice(0, 7); const path = FS(`logs/${month}/${tier}/${currentDay}/${tier}-${number}.log.json`); if (await path.exists()) { return JSON.parse(path.readSync()).log; } return null; } } // 100 iterations is enough to search 2**100 days, which is around 1e30 days // for comparison, a millennium is 365000 days throw new Error(`Infinite loop looking for ${tier}-${number}`); } }; export const LogViewer = new class { async day(roomid: RoomID, day: string, opts?: string) { const month = LogReader.getMonth(day); let buf = `

` + `◂ All logs / ` + `${roomid} / ` + `${month} / ` + `${day}

${opts ? `Options in use: ${opts}` : ''}
`; const roomLog = await LogReader.get(roomid); if (!roomLog) { buf += `

Room "${roomid}" doesn't exist

`; return this.linkify(buf); } const prevDay = LogReader.prevDay(day); const prevRoomid = `view-chatlog-${roomid}--${prevDay}${opts ? `--${opts}` : ''}`; buf += `


${prevDay}

` + `
`; const stream = await roomLog.getLog(day); if (!stream) { buf += `

Room "${roomid}" doesn't have logs for ${day}

`; } else { for await (const line of stream.byLine()) { buf += this.renderLine(line, opts); } } buf += `
`; if (day !== LogReader.today()) { const nextDay = LogReader.nextDay(day); const nextRoomid = `view-chatlog-${roomid}--${nextDay}${opts ? `--${opts}` : ''}`; buf += `

${nextDay}

`; } buf += ``; return this.linkify(buf); } async battle(tier: string, number: number, context: PageContext) { if (number > Rooms.global.lastBattle) { throw new Chat.ErrorMessage(`That battle cannot exist, as the number has not been used.`); } const roomid = `battle-${tier}-${number}` as RoomID; context.send(`

Locating battle logs for the battle ${tier}-${number}...

`); const log = await PM.query({ queryType: 'battlesearch', roomid: toID(tier), search: number, }); if (!log) return context.send(this.error("Logs not found.")); const {connection} = context; context.close(); connection.sendTo( roomid, `|init|battle\n|title|[Battle Log] ${tier}-${number}\n${log.join('\n')}` ); connection.sendTo(roomid, `|expire|This is a battle log.`); } renderLine(fullLine: string, opts?: string) { if (!fullLine) return ``; if (opts === 'txt') return Utils.html`
${fullLine}
`; let timestamp = fullLine.slice(0, opts ? 8 : 5); let line; if (/^[0-9:]+$/.test(timestamp)) { line = fullLine.charAt(9) === '|' ? fullLine.slice(10) : '|' + fullLine.slice(9); } else { timestamp = ''; line = '!NT|'; } if (opts !== 'all' && ( line.startsWith(`userstats|`) || line.startsWith('J|') || line.startsWith('L|') || line.startsWith('N|') )) return ``; const cmd = line.slice(0, line.indexOf('|')); if (opts?.includes('onlychat')) { if (cmd !== 'c') return ''; if (opts.includes('txt')) return `
${Utils.escapeHTML(fullLine)}
`; } switch (cmd) { case 'c': { const [, name, message] = Utils.splitFirst(line, '|', 2); if (name.length <= 1) { return `
[${timestamp}] ${Chat.formatText(message)}
`; } if (message.startsWith(`/log `)) { return `
[${timestamp}] ${Chat.formatText(message.slice(5))}
`; } if (message.startsWith(`/raw `)) { return `
${message.slice(5)}
`; } if (message.startsWith(`/uhtml `) || message.startsWith(`/uhtmlchange `)) { if (message.startsWith(`/uhtmlchange `)) return ``; if (opts !== 'all') return `
[uhtml box hidden]
`; return `
${message.slice(message.indexOf(',') + 1)}
`; } const group = !name.startsWith(' ') ? name.charAt(0) : ``; return `
` + Utils.html`[${timestamp}] ${group}${name.slice(1)}: ` + `${Chat.formatText(message)}` + `
`; } case 'html': case 'raw': { const [, html] = Utils.splitFirst(line, '|', 1); return `
${html}
`; } case 'uhtml': case 'uhtmlchange': { if (cmd !== 'uhtml') return ``; const [, , html] = Utils.splitFirst(line, '|', 2); return `
${html}
`; } case '!NT': return `
${Utils.escapeHTML(fullLine)}
`; case '': return `
[${timestamp}] ${Utils.escapeHTML(line.slice(1))}
`; default: return `
[${timestamp}] ${'|' + Utils.escapeHTML(line)}
`; } } async month(roomid: RoomID, month: string) { let buf = `

` + `◂ All logs / ` + `${roomid} / ` + `${month}


`; const roomLog = await LogReader.get(roomid); if (!roomLog) { buf += `

Room "${roomid}" doesn't exist

`; return this.linkify(buf); } const prevMonth = LogReader.prevMonth(month); buf += `


${prevMonth}

`; const days = await roomLog.listDays(month); if (!days.length) { buf += `

Room "${roomid}" doesn't have logs in ${month}

`; return this.linkify(buf); } else { for (const day of days) { buf += `

- ${day} `; for (const opt of ['txt', 'onlychat', 'all', 'txt-onlychat']) { buf += ` (${opt}) `; } buf += `

`; } } if (!LogReader.today().startsWith(month)) { const nextMonth = LogReader.nextMonth(month); buf += `

${nextMonth}

`; } buf += ``; return this.linkify(buf); } async room(roomid: RoomID) { let buf = `

` + `◂ All logs / ` + `${roomid}


`; const roomLog = await LogReader.get(roomid); if (!roomLog) { buf += `

Room "${roomid}" doesn't exist

`; return this.linkify(buf); } const months = await roomLog.listMonths(); if (!months.length) { buf += `

Room "${roomid}" doesn't have logs

`; return this.linkify(buf); } for (const month of months) { buf += `

- ${month}

`; } buf += ``; return this.linkify(buf); } async list(user: User, opts?: string) { let buf = `

` + `All logs


`; const categories: {[k: string]: string} = { 'official': "Official", 'normal': "Public", 'hidden': "Hidden", 'secret': "Secret", 'deleted': "Deleted", 'personal': "Personal", 'deletedPersonal': "Deleted Personal", }; const list = await LogReader.listCategorized(user, opts) as {[k: string]: RoomID[]}; if (!list) { buf += `

You must be a staff member of a room to view its logs

`; return buf; } const showPersonalLink = opts !== 'all' && user.can('rangeban'); for (const k in categories) { if (!list[k].length && !(['personal', 'deleted'].includes(k) && showPersonalLink)) { continue; } buf += `

${categories[k]}

`; if (k === 'personal' && showPersonalLink) { if (opts !== 'help') buf += `

- (show all help)

`; if (opts !== 'groupchat') buf += `

- (show all groupchat)

`; } if (k === 'deleted' && showPersonalLink) { if (opts !== 'deleted') buf += `

- (show deleted)

`; } for (const roomid of list[k]) { buf += `

- ${roomid}

`; } } buf += ``; return this.linkify(buf); } error(message: string) { return `

${message}

`; } linkify(buf: string) { return buf.replace(/

Linecounts on `; buf += `${roomid}${user ? ` for the user ${user}` : ` (top ${MAX_TOPUSERS})`}

`; buf += `Month: ${month}:
`; const nextMonth = LogReader.nextMonth(month); const prevMonth = LogReader.prevMonth(month); if (FS(`logs/chat/${roomid}/${prevMonth}`).existsSync()) { buf += `
Previous month`; } if (FS(`logs/chat/${roomid}/${nextMonth}`).existsSync()) { buf += ` Next month`; } buf += `
    `; if (user) { const sortedDays = Object.keys(results).sort((a, b) => ( new Date(b).getTime() - new Date(a).getTime() )); for (const day of sortedDays) { const dayResults = results[day][user]; if (isNaN(dayResults)) continue; buf += `
  1. [${day}]: `; buf += `${Chat.count(dayResults, 'lines')}
  2. `; } } else { // squish the results together const totalResults: {[k: string]: number} = {}; for (const date in results) { for (const userid in results[date]) { if (!totalResults[userid]) totalResults[userid] = 0; totalResults[userid] += results[date][userid]; } } const resultKeys = Object.keys(totalResults); const sortedResults = resultKeys.sort((a, b) => ( totalResults[b] - totalResults[a] )).slice(0, MAX_TOPUSERS); for (const userid of sortedResults) { buf += `
  3. ${userid}: `; buf += `${Chat.count(totalResults[userid], 'lines')}
  4. `; } } buf += ``; return LogViewer.linkify(buf); } async runSearch( context: PageContext, search: string, roomid: RoomID, date: string | null, limit: number | null ) { context.title = `[Search] [${roomid}] ${search}`; if (!['ripgrep', 'fs'].includes(Config.chatlogreader)) { throw new Error(`Config.chatlogreader must be 'fs' or 'ripgrep'.`); } context.send( `

    Running a chatlog search for "${search}" on room ${roomid}` + (date ? date !== 'all' ? `, on the date "${date}"` : ', on all dates' : '') + `.

    ` ); const response = await PM.query({search, roomid, date, limit, queryType: 'search'}); return context.send(response); } async runLinecountSearch(context: PageContext, roomid: RoomID, month: string, user?: ID) { context.send( `

    Searching linecounts on room ${roomid}${user ? ` for the user ${user}` : ''}.

    ` ); const results = await PM.query({roomid, date: month, search: user, queryType: 'linecount'}); context.send(results); } async sharedBattles(userids: string[]) { let buf = `Logged shared battles between the users ${userids.join(', ')}`; const results: string[] = await PM.query({ queryType: 'sharedsearch', search: userids, }); if (!results.length) { buf += `:
    None found.`; return buf; } buf += ` (${results.length}):
    `; buf += results.map(id => `${id}`).join(', '); return buf; } } export class FSLogSearcher extends Searcher { results: number; constructor() { super(); this.results = 0; } async searchLinecounts(roomid: RoomID, month: string, user?: ID) { const directory = FS(`logs/chat/${roomid}/${month}`); if (!directory.existsSync()) { throw new Chat.ErrorMessage(`Logs for month '${month}' do not exist on room ${roomid}.`); } const files = await directory.readdir(); const results: {[date: string]: {[userid: string]: number}} = {}; for (const file of files) { const day = file.slice(0, -4); const stream = FS(`logs/chat/${roomid}/${month}/${file}`).createReadStream(); for await (const line of stream.byLine()) { const parts = line.split('|').map(toID); const id = parts[2]; if (!id) continue; if (parts[1] === 'c') { if (user && id !== user) continue; if (!results[day]) results[day] = {}; if (!results[day][id]) results[day][id] = 0; results[day][id]++; } } } return this.renderLinecountResults(results, roomid, month, user); } searchLogs(roomid: RoomID, search: string, limit?: number | null, date?: string | null) { if (!date) date = Chat.toTimestamp(new Date()).split(' ')[0].slice(0, -3); const isAll = (date === 'all'); const isYear = (date.length === 4); const isMonth = (date.length === 7); if (!limit || limit > MAX_RESULTS) limit = MAX_RESULTS; if (isAll) { return this.runYearSearch(roomid, null, search, limit); } else if (isYear) { date = date.substr(0, 4); return this.runYearSearch(roomid, date, search, limit); } else if (isMonth) { date = date.substr(0, 7); return this.runMonthSearch(roomid, date, search, limit); } else { return Promise.resolve(LogViewer.error("Invalid date.")); } } async fsSearchDay(roomid: RoomID, day: string, search: string, limit?: number | null) { if (!limit || limit > MAX_RESULTS) limit = MAX_RESULTS; const text = await LogReader.read(roomid, day, limit); if (!text) return []; const lines = text.split('\n'); const matches: SearchMatch[] = []; const searchTerms = search.split('+').filter(Boolean); const searchTermRegexes: RegExp[] = []; for (const searchTerm of searchTerms) { if (searchTerm.startsWith('user-')) { const id = toID(searchTerm.slice(5)); searchTermRegexes.push(new RegExp(`\\|c\\|${this.constructUserRegex(id)}\\|`, 'i')); continue; } searchTermRegexes.push(new RegExp(searchTerm, 'i')); } function matchLine(line: string) { return searchTermRegexes.every(term => term.test(line)); } for (const [i, line] of lines.entries()) { if (matchLine(line)) { matches.push([ lines[i - 2], lines[i - 1], line, lines[i + 1], lines[i + 2], ]); if (matches.length > limit) break; } } return matches; } renderDayResults(results: {[day: string]: SearchMatch[]}, roomid: RoomID) { const renderResult = (match: SearchMatch) => { this.results++; return ( LogViewer.renderLine(match[0]) + LogViewer.renderLine(match[1]) + `
    ${LogViewer.renderLine(match[2])}
    ` + LogViewer.renderLine(match[3]) + LogViewer.renderLine(match[4]) ); }; let buf = ``; for (const day in results) { const dayResults = results[day]; const plural = dayResults.length !== 1 ? "es" : ""; buf += `
    ${dayResults.length} match${plural} on `; buf += `${day}

    `; buf += `

    ${dayResults.filter(Boolean).map(result => renderResult(result)).join(`


    `)}

    `; buf += `

    `; } return buf; } async fsSearchMonth(opts: ChatlogSearch) { let {limit, room: roomid, date: month, search} = opts; if (!limit || limit > MAX_RESULTS) limit = MAX_RESULTS; const log = await LogReader.get(roomid); if (!log) return {results: {}, total: 0}; const days = await log.listDays(month); const results: {[k: string]: SearchMatch[]} = {}; let total = 0; for (const day of days) { const dayResults = await this.fsSearchDay(roomid, day, search, limit ? limit - total : null); if (!dayResults.length) continue; total += dayResults.length; results[day] = dayResults; if (total > limit) break; } return {results, total}; } /** pass a null `year` to search all-time */ async fsSearchYear(roomid: RoomID, year: string | null, search: string, limit?: number | null) { if (!limit || limit > MAX_RESULTS) limit = MAX_RESULTS; const log = await LogReader.get(roomid); if (!log) return {results: {}, total: 0}; let months = await log.listMonths(); months = months.reverse(); const results: {[k: string]: SearchMatch[]} = {}; let total = 0; for (const month of months) { if (year && !month.includes(year)) continue; const monthSearch = await this.fsSearchMonth({room: roomid, date: month, search, limit}); const {results: monthResults, total: monthTotal} = monthSearch; if (!monthTotal) continue; total += monthTotal; Object.assign(results, monthResults); if (total > limit) break; } return {results, total}; } async runYearSearch(roomid: RoomID, year: string | null, search: string, limit: number) { const {results, total} = await this.fsSearchYear(roomid, year, search, limit); if (!total) { return LogViewer.error(`No matches found for ${search} on ${roomid}.`); } let buf = ''; if (year) { buf += `

    Searching year: ${year}:

    `; } else { buf += `

    Searching all logs:

    `; } buf += this.renderDayResults(results, roomid); if (total > limit) { // cap is met buf += `
    Max results reached, capped at ${total > limit ? limit : MAX_RESULTS}`; buf += `
    `; if (total < MAX_RESULTS) { buf += ``; buf += `
    `; } } this.results = 0; return buf; } async runMonthSearch(roomid: RoomID, month: string, search: string, limit: number, year = false) { const {results, total} = await this.fsSearchMonth({room: roomid, date: month, search, limit}); if (!total) { return LogViewer.error(`No matches found for ${search} on ${roomid}.`); } let buf = ( `
    Searching for "${search}" in ${roomid} (${month}):
    ` ); buf += this.renderDayResults(results, roomid); if (total > limit) { // cap is met & is not being used in a year read buf += `
    Max results reached, capped at ${limit}`; buf += `
    `; if (total < MAX_RESULTS) { buf += ``; buf += `
    `; } } buf += `
    `; this.results = 0; return buf; } async getSharedBattles(userids: string[]) { const months = FS("logs/").readdirSync().filter(f => !isNaN(new Date(f).getTime())); const results: string[] = []; for (const month of months) { const tiers = await FS(`logs/${month}`).readdir(); for (const tier of tiers) { const days = await FS(`logs/${month}/${tier}/`).readdir(); for (const day of days) { const battles = await FS(`logs/${month}/${tier}/${day}`).readdir(); for (const battle of battles) { const content = JSON.parse(FS(`logs/${month}/${tier}/${day}/${battle}`).readSync()); const players = [content.p1, content.p2].map(toID); if (players.every(p => userids.includes(p))) { const battleName = battle.slice(0, -9); results.push(battleName); } } } } } return results; } } export class RipgrepLogSearcher extends Searcher { async ripgrepSearchMonth(opts: ChatlogSearch) { let {raw, search, room: roomid, date: month, args} = opts; let results: string[]; let count = 0; if (!raw) { search = this.constructSearchRegex(search); } const resultSep = args?.includes('-m') ? '--' : '\n'; try { const options = [ '-e', search, `logs/chat/${roomid}/${month}`, '-i', ]; if (args) { options.push(...args); } const {stdout} = await exec(['rg', ...options], { maxBuffer: MAX_MEMORY, cwd: `${__dirname}/../../`, }); results = stdout.split(resultSep); } catch (e) { if (e.code !== 1 && !e.message.includes('stdout maxBuffer') && !e.message.includes('No such file or directory')) { throw e; // 2 means an error in ripgrep } if (e.stdout) { results = e.stdout.split(resultSep); } else { results = []; } } count += results.length; return {results, count}; } async searchLogs( roomid: RoomID, search: string, limit?: number | null, date?: string | null ) { if (date) { // if it's more than 7 chars, assume it's a month if (date.length > 7) date = date.substr(0, 7); // if it's less, assume they were trying a year else if (date.length < 7) date = date.substr(0, 4); } const months = (date && toID(date) !== 'all' ? [date] : await new LogReaderRoom(roomid).listMonths()).reverse(); let count = 0; let results: string[] = []; if (!limit || limit > MAX_RESULTS) limit = MAX_RESULTS; if (!date) date = 'all'; const originalSearch = search; const userRegex = /user-(.[a-zA-Z0-9]*)/gi; const user = userRegex.exec(search)?.[0]?.slice(5); const userSearch = user ? `the user '${user}'` : null; if (userSearch) { const id = toID(user); const rest = search.replace(userRegex, '') .split('-') .filter(Boolean) .map(str => `.*${Utils.escapeRegex(str)}`) .join(''); search = `\\|c\\|${this.constructUserRegex(id)}\\|${rest}`; } while (count < MAX_RESULTS) { const month = months.shift(); if (!month) break; const output = await this.ripgrepSearchMonth({ room: roomid, search, date: month, limit, args: [`-m`, `${limit}`, '-C', '3', '--engine=auto'], raw: !!userSearch, }); results = results.concat(output.results); count += output.count; } if (count > MAX_RESULTS) { const diff = count - MAX_RESULTS; results = results.slice(0, -diff); } return this.renderSearchResults(results, roomid, search, limit, date, originalSearch); } renderSearchResults( results: string[], roomid: RoomID, search: string, limit: number, month?: string | null, originalSearch?: string | null ) { results = results.filter(Boolean); if (results.length < 1) return LogViewer.error('No results found.'); let exactMatches = 0; let curDate = ''; if (limit > MAX_RESULTS) limit = MAX_RESULTS; const useOriginal = originalSearch && originalSearch !== search; const searchRegex = new RegExp(useOriginal ? search : this.constructSearchRegex(search), "i"); const sorted = results.sort((aLine, bLine) => { const [aName] = aLine.split('.txt'); const [bName] = bLine.split('.txt'); const aDate = new Date(aName.split('/').pop()!); const bDate = new Date(bName.split('/').pop()!); return bDate.getTime() - aDate.getTime(); }).map(chunk => chunk.split('\n').map(rawLine => { if (exactMatches > limit || !toID(rawLine)) return null; // return early so we don't keep sorting const sep = rawLine.includes('.txt-') ? '.txt-' : '.txt:'; const [name, text] = rawLine.split(sep); let line = LogViewer.renderLine(text, 'all'); if (!line || name.includes('today')) return null; // gets rid of some edge cases / duplicates let date = name.replace(`logs/chat/${roomid}${toID(month) === 'all' ? '' : `/${month}`}`, '').slice(9); if (searchRegex.test(rawLine)) { if (++exactMatches > limit) return null; line = `
    ${line}
    `; } if (curDate !== date) { curDate = date; date = `
    [${date}]`; } else { date = ''; } return `${date} ${line}`; }).filter(Boolean).join(' ')).filter(Boolean); let buf = `
    Results on ${roomid} for ${originalSearch ? originalSearch : search}:`; buf += limit ? ` ${exactMatches} (capped at ${limit})` : ''; buf += `
    `; buf += sorted.join('
    '); if (limit) { buf += `

    Capped at ${limit}.
    `; buf += ``; buf += `
    `; } return buf; } async searchLinecounts(room: RoomID, month: string, user?: ID) { // don't need to check if logs exist since ripgrepSearchMonth does that // eslint-disable-next-line no-useless-escape const regexString = user ? `\\|c\\|${this.constructUserRegex(user)}\\|` : `\\|c\\|`; const args: string[] = user ? ['--count'] : []; const {results: rawResults} = await this.ripgrepSearchMonth({ search: regexString, raw: true, date: month, room, args, }); if (!rawResults.length) return LogViewer.error(`No results found.`); const results: {[k: string]: {[userid: string]: number}} = {}; for (const fullLine of rawResults) { const [data, line] = fullLine.split('.txt:'); const date = data.split('/').pop()!; if (!results[date]) results[date] = {}; if (!toID(date)) continue; if (user) { if (!results[date][user]) results[date][user] = 0; const parsed = parseInt(line); results[date][user] += isNaN(parsed) ? 0 : parsed; } else { const parts = line?.split('|').map(toID); if (!parts || parts[1] !== 'c') continue; const id = parts[2]; if (!id) continue; if (!results[date][id]) results[date][id] = 0; results[date][id]++; } } return this.renderLinecountResults(results, room, month, user); } async getSharedBattles(userids: string[]) { const regexString = userids.map(id => `(?=.*?("p(1|2)":"${[...id].join('[^a-zA-Z0-9]*')}[^a-zA-Z0-9]*"))`).join(''); const results: string[] = []; try { const {stdout} = await exec(['rg', '-e', regexString, '-i', '-tjson', 'logs/', '-P']); for (const line of stdout.split('\n')) { const [name] = line.split(':'); const battleName = name.split('/').pop()!; results.push(battleName.slice(0, -9)); } } catch (e) { if (e.code !== 1) throw e; } return results.filter(Boolean); } } export const LogSearcher: Searcher = new (Config.chatlogreader === 'ripgrep' ? RipgrepLogSearcher : FSLogSearcher)(); export const PM = new QueryProcessManager(module, async data => { try { const {date, search, roomid, limit, queryType} = data; switch (queryType) { case 'linecount': return LogSearcher.searchLinecounts(roomid, date, search); case 'search': return LogSearcher.searchLogs(roomid, search, limit, date); case 'sharedsearch': return LogSearcher.getSharedBattles(search); case 'battlesearch': return LogReader.findBattleLog(roomid, search); default: return LogViewer.error(`Config.chatlogreader is not configured.`); } } catch (e) { if (e.name?.endsWith('ErrorMessage')) { return LogViewer.error(e.message); } Monitor.crashlog(e, 'A chatlog search query', data); return LogViewer.error(`Sorry! Your chatlog search crashed. We've been notified and will fix this.`); } }, CHATLOG_PM_TIMEOUT); if (!PM.isParentProcess) { // This is a child process! global.Config = Config; global.Monitor = { crashlog(error: Error, source = 'A chatlog search process', details: AnyObject | null = null) { const repr = JSON.stringify([error.name, error.message, source, details]); process.send!(`THROW\n@!!@${repr}\n${error.stack}`); }, }; global.Dex = Dex; global.toID = Dex.toID; global.Chat = Chat; process.on('uncaughtException', err => { if (Config.crashguard) { Monitor.crashlog(err, 'A chatlog search child process'); } }); // eslint-disable-next-line no-eval Repl.start('chatlog', cmd => eval(cmd)); } else { PM.spawn(MAX_PROCESSES); } const accessLog = FS(`logs/chatlog-access.txt`).createAppendStream(); export const pages: PageTable = { async chatlog(args, user, connection) { if (!user.named) return Rooms.RETRY_AFTER_LOGIN; let [roomid, date, opts] = Utils.splitFirst(args.join('-'), '--', 2) as [RoomID, string | undefined, string | undefined]; if (date) date = date.trim(); if (!roomid || roomid.startsWith('-')) { this.title = '[Logs]'; return LogViewer.list(user, roomid?.slice(1)); } // permission check const room = Rooms.get(roomid); if (!user.trusted) { if (room) { this.checkCan('declare', null, room); } else { return this.errorReply(`Access denied.`); } } if (!user.can('rangeban')) { // Some chatlogs can only be viewed by upper staff if (roomid.startsWith('spl') && roomid !== 'splatoon') { return this.errorReply("SPL team discussions are super secret."); } if (roomid.startsWith('wcop')) { return this.errorReply("WCOP team discussions are super secret."); } if (UPPER_STAFF_ROOMS.includes(roomid)) { return this.errorReply("Upper staff rooms are super secret."); } } if (room) { if (!user.can('lock') || room.settings.isPrivate === 'hidden' && !room.checkModjoin(user)) { if (!room.persist) return this.errorReply(`Access denied.`); this.checkCan('mute', null, room); } } else { this.checkCan('lock'); } void accessLog.writeLine(`${user.id}: <${roomid}> ${date}`); this.title = '[Logs] ' + roomid; /** null = no limit */ let limit: number | null = null; let search; if (opts?.startsWith('search-')) { let [input, limitString] = opts.split('--limit-'); input = input.slice(7); search = Dashycode.decode(input); if (search.length < 3) return this.errorReply(`That's too short of a search query.`); if (limitString) { limit = parseInt(limitString) || null; } else { limit = 500; } opts = ''; } const isAll = (toID(date) === 'all' || toID(date) === 'alltime'); const parsedDate = new Date(date as string); const validDateStrings = ['all', 'alltime', 'today']; // this is apparently the best way to tell if a date is invalid if (date && isNaN(parsedDate.getTime()) && !validDateStrings.includes(toID(date))) { return this.errorReply(`Invalid date.`); } if (date && search) { return LogSearcher.runSearch(this, search, roomid, isAll ? null : date, limit); } else if (date) { if (date === 'today') { return LogViewer.day(roomid, LogReader.today(), opts); } else if (date.split('-').length === 3) { return LogViewer.day(roomid, parsedDate.toISOString().slice(0, 10), opts); } else { return LogViewer.month(roomid, parsedDate.toISOString().slice(0, 7)); } } else { return LogViewer.room(roomid); } }, roomstats(args, user) { const room = this.extractRoom(); if (room) { this.checkCan('mute', null, room); } else { if (!user.can('bypassall')) { return this.errorReply(`You cannot view logs for rooms that no longer exist.`); } } const [, date, target] = Utils.splitFirst(args.join('-'), '--', 3).map(item => item.trim()); if (isNaN(new Date(date).getTime())) { return this.errorReply(`Invalid date.`); } this.title = `[Log Stats] ${date}`; return LogSearcher.runLinecountSearch(this, room ? room.roomid : args[2] as RoomID, date, toID(target)); }, battlelog(args, user) { const [tierName, battleNum] = args; const tier = toID(tierName); const num = parseInt(battleNum); if (isNaN(num)) return this.errorReply(`Invalid battle number.`); void accessLog.writeLine(`${user.id}: battle-${tier}-${num}`); return LogViewer.battle(tier, num, this); }, async logsaccess(query) { this.checkCan('rangeban'); const type = toID(query.shift()); if (type && !['chat', 'battle', 'all', 'battles'].includes(type)) { return this.errorReply(`Invalid log type.`); } let title = ''; switch (type) { case 'battle': case 'battles': title = 'Battlelog access log'; break; case 'chat': title = 'Chatlog access log'; break; default: title = 'Logs access log'; break; } const userid = toID(query.shift()); let buf = `

    ${title}`; if (userid) buf += ` for ${userid}`; buf += `


      `; const accessStream = FS(`logs/chatlog-access.txt`).createReadStream(); for await (const line of accessStream.byLine()) { const [id, rest] = Utils.splitFirst(line, ': '); if (userid && id !== userid) continue; if (type === 'battle' && !line.includes('battle-')) continue; if (userid) { buf += `
    1. ${rest}
    2. `; } else { buf += `
    3. ${id}: ${rest}
    4. `; } } buf += `
    `; return buf; }, }; export const commands: ChatCommands = { chatlog(target, room, user) { const [tarRoom, ...opts] = target.split(','); const targetRoom = tarRoom ? Rooms.search(tarRoom) : room; const roomid = targetRoom ? targetRoom.roomid : target; return this.parse(`/join view-chatlog-${roomid}--today${opts ? `--${opts.join('--')}` : ''}`); }, chatloghelp() { const strings = [ `/chatlog [optional room], [opts] - View chatlogs from the given room. `, `If none is specified, shows logs from the room you're in. Requires: % @ * # &`, `Supported options:`, `txt - Do not render logs.`, `txt-onlychat - Show only chat lines, untransformed.`, `onlychat - Show only chat lines.`, `all - Show all lines, including userstats and join/leave messages.`, ]; this.runBroadcast(); return this.sendReplyBox(strings.join('
    ')); }, sl: 'searchlogs', logsearch: 'searchlogs', searchlog: 'searchlogs', searchlogs(target, room) { target = target.trim(); const args = target.split(',').map(item => item.trim()); if (!target) return this.parse('/help searchlogs'); let date = 'all'; const searches: string[] = []; let limit = '500'; for (const arg of args) { if (arg.startsWith('room:')) { const id = arg.slice(5); room = Rooms.search(id as RoomID) as Room | null; if (!room) { return this.errorReply(`Room "${id}" not found.`); } } else if (arg.startsWith('limit:')) { limit = arg.slice(6); } else if (arg.startsWith('date:')) { date = arg.slice(5); } else if (arg.startsWith('user:')) { args.push(`user-${toID(arg.slice(5))}`); } else { searches.push(arg); } } if (!room) { return this.parse(`/help searchlogs`); } return this.parse( `/join view-chatlog-${room.roomid}--${date}--search-` + `${Dashycode.encode(searches.join('+'))}--limit-${limit}` ); }, searchlogshelp() { const buffer = `
    /searchlogs [arguments]: ` + `searches logs in the current room using the [arguments].` + `A room can be specified using the argument room: [roomid]. Defaults to the room it is used in.
    ` + `A limit can be specified using the argument limit: [number less than or equal to 3000]. Defaults to 500.
    ` + `A date can be specified in ISO (YYYY-MM-DD) format using the argument date: [month] (for example, date: 2020-05). Defaults to searching all logs.
    ` + `If you provide a user argument in the form user:username, it will search for messages (that match the other arguments) only from that user` + `All other arguments will be considered part of the search ` + `(if more than one argument is specified, it searches for lines containing all terms).
    ` + "Requires: % @ # &
    "; return this.sendReplyBox(buffer); }, topusers: 'linecount', roomstats: 'linecount', linecount(target, room, user) { let [roomid, month, userid] = target.split(',').map(item => item.trim()); const tarRoom = roomid ? Rooms.search(roomid) : room; if (!tarRoom) return this.errorReply(`You must specify a valid room.`); if (!month) month = LogReader.getMonth(); return this.parse(`/join view-roomstats-${tarRoom.roomid}--${month}--${toID(userid)}`); }, linecounthelp: [ `/topusers OR /linecount [room], [month], [userid] - View room stats in the given [room].`, `If a user is provided, searches only for that user, else the top 100 users are shown.`, `Requires: % @ # &`, ], slb: 'sharedloggedbattles', async sharedloggedbattles(target, room, user) { this.checkCan('lock'); const targets = target.split(',').map(toID).filter(Boolean); if (targets.length < 2 || targets.length > 2) { return this.errorReply(`Specify two users.`); } const results = await LogSearcher.sharedBattles(targets); if (room?.settings.staffRoom || this.pmTarget?.isStaff) { this.runBroadcast(); } return this.sendReplyBox(results); }, sharedloggedbattleshelp: [ `/sharedloggedbattles OR /slb [user1, user2] - View shared battle logs between user1 and user2`, ], battlelog(target, room, user) { this.checkCan('lock'); target = target.trim(); if (!target) return this.errorReply(`Specify a battle.`); if (target.startsWith('http://')) target = target.slice(7); if (target.startsWith('https://')) target = target.slice(8); if (target.startsWith(`${Config.routes.client}/`)) target = target.slice(Config.routes.client.length + 1); if (target.startsWith(`${Config.routes.replays}/`)) target = `battle-${target.slice(Config.routes.replays.length + 1)}`; if (target.startsWith('psim.us/')) target = target.slice(8); return this.parse(`/join view-battlelog-${target}`); }, logsaccess(target, room, user) { this.checkCan('rangeban'); const [type, userid] = target.split(',').map(toID); return this.parse(`/j view-logsaccess-${type || 'all'}${userid ? `-${userid}` : ''}`); }, };