pokemon-showdown/server/ip-tools.ts
2019-06-29 07:28:58 -07:00

645 lines
19 KiB
TypeScript

/**
* IP Tools
* Pokemon Showdown - http://pokemonshowdown.com/
*
* IPTools file has various tools for IP parsing and IP-based blocking.
*
* These include DNSBLs: DNS-based blackhole lists, which list IPs known for
* running proxies, spamming, or other abuse.
*
* We also maintain our own database of datacenter IP ranges (usually
* proxies). These are taken from https://github.com/client9/ipcat
* but include our own database as well.
*
* @license MIT
*/
'use strict';
const BLOCKLISTS = ['sbl.spamhaus.org', 'rbl.efnetrbl.org'];
import * as dns from 'dns';
import {FS} from '../lib/fs';
export const IPTools = new class {
dnsblCache = new Map<string, string | null>([
['127.0.0.01', null],
]);
proxyHosts = new Set([
'alexhost.md',
'amazonaws.com',
'anchorfree.com',
'aquanetworks.co.uk',
'aruba.it',
'arubacloud.com',
'arubacloud.de',
'arubacloud.fr',
'as29550.net',
'avenueoftor.com',
'bitcoinwebhosting.net',
'blazingfast.io',
'bnewstoday.com',
'brightnewfutures.net',
'cd-n.net',
'choopa.com',
'choopa.net',
'cloudhosting.lv',
'cloudmosa.com',
'clues.ro',
'colocenter.nl',
'colocrossing.com',
'contaboserver.net',
'cyberghost.ro',
'cyberghostvpn.com',
'darkweb.love',
'dockerapp.io',
'edis.at',
'elostech.cz',
'enahost.com',
'ezzi.net',
'fantastic-host.net',
'forpsi.net',
'freedome-vpn.net',
'frootvpn.com',
'galaxyhostplus.com',
'gigenet.com',
'hide.me',
'hkserverworks.com',
'hostnottingham.co.uk',
'hostwindsdns.com',
'hvvc.us',
'hwng.net',
'ihc.ru',
'ipvanish.com',
'linode.com',
'ltt.ly',
'lunanode.com',
'm247.com',
'mediateam.fi',
'midphase.com',
'mivocloud.com',
'mixskilled.com',
'netdive.xyz',
'networktransit.net',
'northamericancoax.com',
'northghost.com',
'novalayer.net',
'nu-s.net',
'op-net.com',
'openvirtuals.com',
'opera.com',
'pacswitch.com',
'poneytelecom.eu',
'primegraf.com.br',
'privacyfoundation.ch',
'protectedgroup.com',
'quadranet.com',
'ra4wvpn.com',
'redstation.co.uk',
'redstation.net.uk',
'samsnuk.info',
'scaleway.com',
'serverprofi24.com',
'shinyrimsinc.com',
'shpv.eu',
'sl-reverse.com',
'smartwebbrands.com',
'softlayer.com',
'stephost.md',
'terrahost.no',
'time4vps.eu',
'trance.fm',
'tunnelbear.com',
'ubiquityservers.com',
'uaservers.net',
'ukservers.com',
'unmetered.zone',
'upcloud.host',
'vilayer.com',
'voxility.com',
'voxility.net',
'vpnbook.com',
'vultr.com',
'worldstream.nl',
'your-server.de',
'zenmate.com',
]);
residentialHosts = new Set([
'bell.ca',
'bellmts.net',
'bellsouth.net',
'cgocable.net',
'charter.com',
'comcast.net',
'comcast.net.res-nohost',
'comcastbusiness.net',
'cornell.edu',
'cox.net',
'embarqhsd.net',
'frontiernet.net',
'hargray.net',
'itctel.com',
'mlgc.com',
'netins.net',
'optonline.net',
'odu.edu',
'osu.edu',
'qwest.net',
'rogers.com',
'rr.com',
'sbcglobal.net',
'shawcable.net',
'sourcecable.net',
'unc.edu',
'verizon.net',
'videotron.ca',
'virginmobile.ca',
'vt.edu',
'wayport.net',
'windstream.net',
]);
mobileHosts = new Set([
'myvzw.com',
'mycingular.net',
'spcsdns.net',
'tmodns.net',
'tmobile.mobile-nohost',
'tele2.se',
'ideacellular.mobile-nohost',
'as13285.net',
'att.net',
]);
connectionTestCache = new Map<string, boolean>();
async lookup(ip: string) {
const [dnsbl, host] = await Promise.all([
IPTools.queryDnsbl(ip),
IPTools.getHost(ip),
]);
const shortHost = this.shortenHost(host);
const hostType = this.getHostType(shortHost, ip);
return {dnsbl, host, shortHost, hostType};
}
queryDnsblLoop(ip: string, callback: (val: string | null) => void, reversedIpDot: string, index: number) {
if (index >= BLOCKLISTS.length) {
// not in any blocklist
IPTools.dnsblCache.set(ip, null);
callback(null);
return;
}
const blocklist = BLOCKLISTS[index];
dns.resolve4(reversedIpDot + blocklist, (err, addresses) => {
if (!err) {
// blocked
IPTools.dnsblCache.set(ip, blocklist);
callback(blocklist);
return;
}
// not blocked, try next blocklist
IPTools.queryDnsblLoop(ip, callback, reversedIpDot, index + 1);
});
}
/**
* IPTools.queryDnsbl(ip, callback)
*
* Calls callb
* ack(blocklist), where blocklist is the blocklist domain
* if the passed IP is in a blocklist, or null if the IP is not in
* any blocklist.
*
* Return value matches isBlocked when treated as a boolean.
*/
queryDnsbl(ip: string) {
if (!Config.dnsbl) return null;
if (IPTools.dnsblCache.has(ip)) {
return Promise.resolve<string | null>(IPTools.dnsblCache.get(ip) || null);
}
const reversedIpDot = ip.split('.').reverse().join('.') + '.';
return new Promise<string | null>((resolve, reject) => {
IPTools.queryDnsblLoop(ip, resolve, reversedIpDot, 0);
});
}
/*********************************************************
* IP parsing
*********************************************************/
ipToNumber(ip: string) {
if (ip.includes(':') && !ip.includes('.')) {
// IPv6
return -1;
}
if (ip.startsWith('::ffff:')) ip = ip.slice(7);
else if (ip.startsWith('::')) ip = ip.slice(2);
let num = 0;
const parts = ip.split('.');
for (const part of parts) {
num *= 256;
num += parseInt(part);
}
return num;
}
getCidrPattern(cidr: string): [number, number] | null {
if (!cidr) return null;
const index = cidr.indexOf('/');
if (index <= 0) {
return [IPTools.ipToNumber(cidr), IPTools.ipToNumber(cidr)];
}
const low = IPTools.ipToNumber(cidr.slice(0, index));
const bits = parseInt(cidr.slice(index + 1));
// fun fact: IPTools fails if bits <= 1 because JavaScript
// does << with signed int32s.
const high = low + (1 << (32 - bits)) - 1;
return [low, high];
}
getRangePattern(range: string): [number, number] | null {
if (!range) return null;
const index = range.indexOf(' - ');
if (index <= 0) {
return [IPTools.ipToNumber(range), IPTools.ipToNumber(range)];
}
const low = IPTools.ipToNumber(range.slice(0, index));
const high = IPTools.ipToNumber(range.slice(index + 3));
return [low, high];
}
getPattern(str: string): [number, number] | null {
if (!str) return null;
if (str.indexOf(' - ') > 0) return IPTools.getRangePattern(str);
return IPTools.getCidrPattern(str);
}
cidrToPattern(cidr: string | string[]): [number, number][] {
if (!cidr || !cidr.length) {
return [];
}
if (typeof cidr === 'string') {
const pattern = IPTools.getCidrPattern(cidr);
if (!pattern) return [];
return [pattern];
}
return cidr.map(IPTools.getCidrPattern).filter(x => x) as [number, number][];
}
rangeToPattern(range: string | string[]): [number, number][] {
if (!range || !range.length) {
return [];
}
if (typeof range === 'string') {
const pattern = IPTools.getRangePattern(range);
if (!pattern) return [];
return [pattern];
}
return range.map(IPTools.getRangePattern).filter(x => x) as [number, number][];
}
checkPattern(patterns: [number, number][], num: number) {
for (const pattern of patterns) {
if (num >= pattern[0] && num <= pattern[1]) {
return true;
}
}
return false;
}
/**
* Returns a checker function for the passed IP range or array of
* ranges. The checker function returns true if its passed IP is
* in the range.
*/
checker(ranges: string | string[]) {
if (!ranges || !ranges.length) return () => false;
let patterns: [number, number][] = [];
if (typeof ranges === 'string') {
const rangePatterns = IPTools.getPattern(ranges);
if (rangePatterns) patterns = [rangePatterns];
} else {
patterns = ranges.map(IPTools.getPattern).filter(x => x) as [number, number][];
}
return (ip: string) => IPTools.checkPattern(patterns, IPTools.ipToNumber(ip));
}
/*********************************************************
* Datacenter parsing
*********************************************************/
urlToHost(url: string) {
if (url.startsWith('http://')) url = url.slice(7);
if (url.startsWith('https://')) url = url.slice(8);
if (url.startsWith('www.')) url = url.slice(4);
const slashIndex = url.indexOf('/');
if (slashIndex > 0) url = url.slice(0, slashIndex);
return url;
}
datacenters: [number, number, string][] = [];
async loadDatacenters() {
const data = await FS('config/datacenters.csv').readIfExists();
const rows = data.split('\n');
const datacenters: [number, number, string][] = [];
for (const row of rows) {
if (!row) continue;
const rowSplit = row.split(',');
const rowData: [number, number, string] = [
IPTools.ipToNumber(rowSplit[0]),
IPTools.ipToNumber(rowSplit[1]),
IPTools.urlToHost(rowSplit[3]),
];
datacenters.push(rowData);
}
IPTools.datacenters = datacenters;
}
/**
* Will not reject; IPs with no RDNS entry will resolve to
* '[byte1].[byte2].unknown-nohost'.
*/
getHost(ip: string) {
return new Promise<string>(resolve => {
if (!ip) {
resolve('');
return;
}
const ipNumber = IPTools.ipToNumber(ip);
if (IPTools.checkPattern(rangeOVHres, ipNumber)) {
resolve('ovh.fr.res-nohost');
return;
}
if (IPTools.checkPattern(rangeWindres, ipNumber)) {
resolve('wind.it.res-nohost');
return;
}
for (const row of IPTools.datacenters) {
if (ipNumber >= row[0] && ipNumber <= row[1]) {
resolve(row[2] + '.proxy-nohost');
return;
}
}
if (
ip.startsWith('106.76.') || ip.startsWith('106.77.') || ip.startsWith('106.78.') ||
ip.startsWith('106.79.') || ip.startsWith('112.110.') || ip.startsWith('27.97.') ||
ip.startsWith('49.15.') || ip.startsWith('49.14.') || ip.startsWith('1.187.')
) {
resolve('ideacellular.mobile-nohost');
return;
}
if (IPTools.checkPattern(rangeTmobile, ipNumber) || ip.startsWith('149.254.')) {
resolve('tmobile.mobile-nohost');
return;
}
if (
IPTools.checkPattern(rangeCenet, ipNumber) || IPTools.checkPattern(rangeQlded, ipNumber) ||
ip.startsWith('153.107.') || IPTools.checkPattern(rangeCathednet, ipNumber)
) {
resolve('edu.au.res-nohost');
return;
}
if (ip.startsWith('179.7.')) {
resolve('claro.com.pe.mobile-nohost');
return;
}
if (ip.startsWith('190.') || IPTools.checkPattern(rangeTelefonica, ipNumber)) {
resolve('telefonica.net.pe.mobile-nohost');
return;
}
if (ip.startsWith('180.191.') || ip.startsWith('112.198.')) {
resolve('globe.com.ph.mobile-nohost');
return;
}
if (ip.startsWith('218.188.') || ip.startsWith('218.189.')) {
resolve('hgc.com.hk.mobile-nohost');
return;
}
if (ip.startsWith('172.242.') || ip.startsWith('172.243.')) {
resolve('viasat.com.mobile-nohost');
return;
}
if (ip.startsWith('201.141.')) {
resolve('cablevision.net.mx.mobile-nohost');
return;
}
if (IPTools.checkPattern(rangeStarhub, ipNumber)) {
resolve('starhub.com.mobile-nohost');
return;
}
if (ip.startsWith('202.12.94.') || ip.startsWith('202.12.95.')) {
resolve('nyp.edu.sg.res-nohost');
return;
}
if (ip.startsWith('64.150.')) {
resolve('illinois.net.res-nohost');
return;
}
if (ip.startsWith('147.129.')) {
resolve('ithaca.edu.res-nohost');
return;
}
if (ip.startsWith('189.204.')) {
resolve('bestel.com.mx.res-nohost');
return;
}
if (IPTools.checkPattern(rangePsci, ipNumber)) {
resolve('psci.net.res-nohost');
return;
}
if (IPTools.checkPattern(rangeOcde, ipNumber)) {
resolve('ocde.us.res-nohost');
return;
}
if (IPTools.checkPattern(rangeIhet, ipNumber)) {
resolve('iu.edu.res-nohost');
return;
}
if (IPTools.checkPattern(rangeTimcelular, ipNumber)) {
resolve('tim.com.br.mobile-nohost');
return;
}
if (ip.startsWith('121.54.')) {
resolve('smart.com.ph.mobile-nohost');
return;
}
if (ip.startsWith('179.52.') || ip.startsWith('179.53.')) {
resolve('codetel.net.do.mobile-nohost');
return;
}
if (ip.startsWith('46.16.36.')) {
resolve('anchorfree.proxy-nohost');
return;
}
if (
ip.startsWith('198.144.104.') || ip.startsWith('198.47.115.') || ip.startsWith('199.255.215.') ||
ip.startsWith('204.14.76.') || ip.startsWith('204.14.77.') || ip.startsWith('204.14.78.') ||
ip.startsWith('204.14.79.') || ip.startsWith('205.164.32.') || ip.startsWith('209.73.132.') ||
ip.startsWith('209.73.151.') || ip.startsWith('216.172.135.') || ip.startsWith('46.16.34.') ||
ip.startsWith('46.16.35.') || ip.startsWith('50.117.45.') || ip.startsWith('63.141.198.') ||
ip.startsWith('63.141.199.') || ip.startsWith('74.115.1.') || ip.startsWith('74.115.5.') ||
ip.startsWith('85.237.197.') || ip.startsWith('85.237.222.')
) {
resolve('anchorfree.proxy-nohost');
return;
}
if (ip === '127.0.0.1') {
resolve('localhost');
return;
}
dns.reverse(ip, (err, hosts) => {
if (err) {
resolve('' + ip.split('.').slice(0, 2).join('.') + '.unknown-nohost');
return;
}
if (!hosts || !hosts[0]) {
if (ip.startsWith('50.')) {
resolve('comcast.net.res-nohost');
} else if (IPTools.checkPattern(rangeTelstra, ipNumber)) {
resolve('telstra.net.res-nohost');
} else {
this.testConnection(ip, result => {
if (result) {
resolve('' + ip.split('.').slice(0, 2).join('.') + '.proxy-nohost');
} else {
resolve('' + ip.split('.').slice(0, 2).join('.') + '.unknown-nohost');
}
});
}
}
resolve(hosts[0]);
});
});
}
/**
* Does this IP respond to port 80? In theory, proxies are likely to
* respond, while residential connections are likely to reject connections.
*
* Callback is guaranteed to be called exactly once, within a 1000ms
* timeout.
*/
testConnection(ip: string, callback: (result: boolean) => void) {
const cachedValue = this.connectionTestCache.get(ip);
if (cachedValue !== undefined) {
return cachedValue;
}
// Node.js's documentation does not make this easy to write. I discovered
// this behavior by manual testing:
// A successful connection emits 'connect', which you should react to
// with socket.destroy(), which emits 'close'.
// Some IPs instantly reject connections, emitting 'error' followed
// immediately by 'close'.
// Some IPs just never respond, leaving you to time out. Node will
// emit the 'timeout' event, but not actually do anything else, leaving
// you to manually use socket.destroy(), which emits 'close'
let connected = false;
const socket = require('net').createConnection({
port: 80,
host: ip,
timeout: 1000,
}, () => {
connected = true;
this.connectionTestCache.set(ip, true);
callback(true);
socket.destroy();
});
socket.on('error', () => {});
socket.on('timeout', () => socket.destroy());
socket.on('close', () => {
if (!connected) {
this.connectionTestCache.set(ip, false);
callback(false);
}
});
}
shortenHost(host: string) {
if (host.slice(-7) === '-nohost') return host;
let dotLoc = host.lastIndexOf('.');
const tld = host.slice(dotLoc);
if (tld === '.uk' || tld === '.au' || tld === '.br') dotLoc = host.lastIndexOf('.', dotLoc - 1);
dotLoc = host.lastIndexOf('.', dotLoc - 1);
return host.slice(dotLoc + 1);
}
/**
* Host types:
* - 'res' - normal residential ISP
* - 'shared' - like res, but shared among many people: bans will have collateral damage
* - 'mobile' - like res, but unstable IP (IP bans don't work)
* - 'proxy' - datacenters, VPNs, proxy services, other untrustworthy sources
* (note that bots will usually be hosted on these)
* - 'res?' - likely res, but host not specifically whitelisted
* - 'unknown' - no rdns entry, treat with suspicion
*/
getHostType(host: string, ip: string) {
if (Punishments.sharedIps.has(ip)) {
return 'shared';
}
if (host === 'he.net.proxy-nohost') {
// Hurricane Electric has an annoying habit of having residential
// internet and datacenters on the same IP ranges - we get a lot of
// legitimate users as well as spammers on VPNs from HE.
// This splits the difference and treats it like any other unknown IP.
return 'unknown';
}
// There were previously special cases for
// 'digitalocean.proxy-nohost', 'servihosting.es.proxy-nohost'
// DO is commonly used to host bots; I don't know who whitelisted
// servihosting but I assume for a similar reason. This isn't actually
// tenable; any service that can host bots can and does also host proxies.
if (this.proxyHosts.has(host) || host.endsWith('.proxy-nohost')) {
return 'proxy';
}
if (this.residentialHosts.has(host) || host.endsWith('.res-nohost')) {
return 'res';
}
if (this.mobileHosts.has(host) || host.endsWith('.mobile-nohost')) {
return 'mobile';
}
if (/^ip-[0-9]+-[0-9]+-[0-9]+\.net$/.test(host) || /^ip-[0-9]+-[0-9]+-[0-9]+\.eu$/.test(host)) {
// OVH
return 'proxy';
}
if ([
'114.161.93.156', '121.129.20.202', '122.45.113.125', '128.68.107.201', '150.31.37.122', '176.202.180.98', '184.22.240.178', '185.127.25.192', '185.234.219.117', '199.249.230.69', '39.111.5.170', '58.124.244.202', '59.8.242.27', '61.121.64.114', '78.85.106.214', '89.110.59.95', '38.132.116.198', '38.132.120.92', '113.197.177.61', '38.132.120.92', '126.227.135.176', '171.251.45.151', '111.105.163.211', '119.30.32.155', '176.124.96.196', '93.87.75.118', '87.247.111.222', '244.242.108.51', '72.252.4.161', '78.85.17.243', '80.250.14.236', '83.142.197.99', '89.147.80.2', '91.197.189.62', '103.78.54.180', '106.120.14.39', '109.122.80.234', '115.85.65.146', '118.97.55.65', '185.217.160.184', '200.35.43.89', '202.62.61.119', '203.83.183.11', '84.245.104.164', '158.195.148.169',
].includes(ip)) {
return 'proxy';
}
if (host.endsWith('.unknown-nohost')) {
// rdns entry doesn't exist, and IP doesn't respond to a probe on port 80
return 'unknown';
}
// rdns entry exists but is unrecognized
return 'res?';
}
};
const rangeTmobile = IPTools.cidrToPattern('172.32.0.0/11');
const rangeCenet = IPTools.cidrToPattern('27.111.64.0/21');
const rangeQlded = IPTools.cidrToPattern('203.104.0.0/20');
const rangeCathednet = IPTools.cidrToPattern('180.95.40.0/21');
const rangeTelefonica = IPTools.cidrToPattern('181.64.0.0/14');
const rangeStarhub = IPTools.cidrToPattern([
'27.125.128.0/18', '58.96.192.0/18', '101.127.0.0/17', '116.88.0.0/17', '122.11.192.0/18', '182.19.128.0/17', '182.55.0.0/16', '183.90.0.0/17', '203.116.122.0/23',
]);
const rangeTelstra = IPTools.cidrToPattern('101.160.0.0/11');
const rangePsci = IPTools.cidrToPattern(['96.31.192.0/20', '209.239.96.0/20', '216.49.96.0/19']);
const rangeOcde = IPTools.cidrToPattern(['104.249.64.0/18', '209.232.144.0/20', '216.100.88.0/21']);
const rangeIhet = IPTools.cidrToPattern('199.8.0.0/16');
const rangeTimcelular = IPTools.cidrToPattern('191.128.0.0/12');
const rangeOVHres = IPTools.rangeToPattern([
'109.190.0.0 - 109.190.63.255', '109.190.64.0 - 109.190.127.255', '109.190.128.0 - 109.190.191.255', '109.190.192.0 - 109.190.255.255', '151.80.228.0 - 151.80.228.255', '178.32.37.0 - 178.32.37.255', '178.33.101.0 - 178.33.101.255', '185.15.68.0 - 185.15.69.255', '185.15.70.0 - 185.15.71.255',
]);
const rangeWindres = IPTools.rangeToPattern([
'151.3.0.0 - 151.79.255.255', '151.81.0.0 - 151.84.255.255', '151.93.0.0 - 151.93.255.255', '151.95.0.0 - 151.95.255.255', '176.206.0.0 - 176.207.255.255',
]);
export default IPTools;