using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.IO; using System.Net.Sockets; using System.Threading; using MySql.Data.MySqlClient; using System.Configuration; using PkmnFoundations.Data; using PkmnFoundations.Support; using System.Data; using System.Net.Security; using System.Security.Cryptography.X509Certificates; namespace PkmnFoundations { public class BvCrawler5 { public static void Main(string[] args) { m_upload_dir = ConfigurationManager.AppSettings["pkmnFoundationsBoxUpload5Dir"]; Console.WriteLine("Pokémon BW/BW2 Battle Video Crawler by mm201"); int pid = 330241763; // White 1 Jenny (?) Directory.CreateDirectory(String.Format("{0}", m_upload_dir)); DateTime last_top30 = DateTime.MinValue; DateTime last_top_link = DateTime.MinValue; DateTime last_top_subway = DateTime.MinValue; while (true) { ulong videoId; try { using (MySqlConnection db = CreateConnection()) { db.Open(); videoId = DequeueVideo(db); db.Close(); } } catch (Exception ex) { // haven't touched the server, sleep short LogError(ex); Thread.Sleep(1000 * 1); continue; } if (videoId == 0) { try { if (last_top30 < DateTime.Now.AddMinutes(-60)) { last_top30 = DateTime.Now; QueueSpecial(pid, SearchSpecial.Latest30); continue; } if (last_top_link < DateTime.Now.AddMinutes(-120)) { last_top_link = DateTime.Now; QueueSpecial(pid, SearchSpecial.TopLinkBattles); continue; } if (last_top_subway < DateTime.Now.AddMinutes(-120)) { last_top_subway = DateTime.Now; QueueSpecial(pid, SearchSpecial.TopSubwayBattles); continue; } else if (RunSearch(pid)) { continue; } } catch (Exception ex) { LogError(ex); Thread.Sleep(1000 * 60); continue; } Console.WriteLine("Nothing to do. Idling 1 minute."); Thread.Sleep(1000 * 60); continue; } String formatted = FormatVideoId(videoId); String filename = String.Format("{0}\\{1}.bin", m_upload_dir, formatted); if (File.Exists(filename)) { Console.WriteLine("Skipped video {0}. Already present on disk.", formatted); Thread.Sleep(1000 * 1); continue; } byte[] data; try { data = GetBattleVideo(pid, videoId); using (FileStream file = File.Create(filename)) { file.Write(data, 0, data.Length); file.Close(); } } catch (Exception ex) { LogError(ex); Thread.Sleep(1000 * 60); continue; } Console.WriteLine("Successfully saved battle video {0}.", formatted); Thread.Sleep(1000 * 60); } } private static String m_upload_dir; public static String FormatVideoId(ulong videoId) { String number = videoId.ToString("D12"); String[] split = new String[3]; split[0] = number.Substring(0, number.Length - 10); split[1] = number.Substring(number.Length - 10, 5); split[2] = number.Substring(number.Length - 5, 5); return String.Join("-", split); } public static byte[] GetBattleVideo(int pid, ulong videoId) { String formatted = FormatVideoId(videoId); Console.WriteLine("Attempting to retrieve battle video {0} from server.", formatted); byte[] data = new byte[0x14c]; MemoryStream request = new MemoryStream(data); request.Write(new byte[4], 0, 4); // length goes here, see end request.Write(new byte[] { 0xf2, 0x55, 0x00, 0x00 }, 0, 4); // request type, sanity 0000 request.Write(BitConverter.GetBytes(pid), 0, 4); // pid, hopefully this doesn't ban me request.Write(new byte[] { 0x17, 0x02 }, 0, 2); // there is some random bytes contained in this sometimes. Could be trainer profile // related, I don't know... // in Genv it's base64 encoded?? request.Write(new byte[] { 0x5a, 0x6f, 0x66, 0x78, 0x74, 0x5a, 0x5a, 0x48, 0x54, 0x66, 0x73, 0x4f, 0x43, 0x78, 0x6d, 0x39, 0x76, 0x57, 0x6c, 0x65, 0x63, 0x51, 0x4d, 0x35, 0x37, 0x62, 0x49, 0x34, 0x70, 0x39, 0x56, 0x61, 0x47, 0x59, 0x41, 0x4a, 0x31, 0x4b, 0x70, 0x2f, 0x72, 0x74, 0x4f, 0x68, 0x6c, 0x42, 0x6c, 0x74, 0x72, 0x48, 0x79, 0x71, 0x53, 0x33, 0x57, 0x70, 0x41, 0x34, 0x34, 0x50, 0x38, 0x73, 0x63, 0x65, 0x5a, 0x4f, 0x59, 0x63, 0x51, 0x4d, 0x2b, 0x67, 0x4e, 0x41, 0x46, 0x34, 0x2b, 0x37, 0x6d, 0x32, 0x74, 0x79, 0x78, 0x4b, 0x59, 0x77, 0x3d, 0x3d }, 0, 0x58); request.Write(new byte[0xda], 0, 0xda); request.Write(BitConverter.GetBytes(videoId), 0, 8); request.Write(new byte[] { 0x64, 0x00, 0x00, 0x00 }, 0, 4); request.Flush(); PutLength(data); byte[] response = Conversation(data); Console.WriteLine("Successfully retrieved {0} byte response for battle video {1}.", response.Length, formatted); return response; } public static bool RunSearch(int pid) { Random rand = new Random(); SearchMetagames[] metagames = (SearchMetagames[])Enum.GetValues(typeof(SearchMetagames)); int metaCount = metagames.Length - 1; // remove None from the list int metaIndex = rand.Next(1, metaCount + 1); SearchMetagames metagame = metagames[metaIndex]; ushort species = (ushort)rand.Next(0, 649); byte country = 0xff; byte region = 0xff; using (MySqlConnection db = CreateConnection()) { db.Open(); if ((long)db.ExecuteScalar( "SELECT Count(*) FROM BattleVideoSearchHistory5 WHERE Metagame = @metagame " + "AND Species = @species AND Country = @country AND Region = @region AND Special = 0", new MySqlParameter("@metagame", (int)metagame), new MySqlParameter("@species", (int)species), new MySqlParameter("@country", (int)country), new MySqlParameter("@region", (int)region)) == 0) { // exact match QueueSearch(pid, SearchSpecial.None, species, metagame, country, region); return true; } DataTable dt; dt = db.ExecuteDataTable("SELECT DISTINCT Metagame, Species, Country, Region " + "FROM BattleVideoSearchHistory5 WHERE Metagame = @metagame AND Special = 0 ORDER BY Species", new MySqlParameter("@metagame", (int)metagame)); if (dt.Rows.Count < 649) { int prevSpecies = 1; foreach (DataRow row in dt.Rows) { if ((int)row["Species"] != prevSpecies) { QueueSearch(pid, SearchSpecial.None, (ushort)(prevSpecies), metagame, country, region); return true; } prevSpecies++; } } dt = db.ExecuteDataTable("SELECT DISTINCT Metagame, Species, Country, Region " + "FROM BattleVideoSearchHistory5 WHERE Species = @species AND Special = 0 ORDER BY Metagame", new MySqlParameter("@species", (int)species)); if (dt.Rows.Count < metaCount) { int prevMeta = 1; foreach (DataRow row in dt.Rows) { if ((int)row["Metagame"] != (int)metagames[prevMeta]) { QueueSearch(pid, SearchSpecial.None, species, metagames[prevMeta], country, region); return true; } prevMeta++; } } dt = db.ExecuteDataTable("SELECT DISTINCT Metagame, Species, Country, Region " + "FROM BattleVideoSearchHistory5 WHERE Special = 0 ORDER BY Metagame, Species"); if (dt.Rows.Count < 649 * metaCount) { int prevSpecies = 1; int prevMeta = 1; foreach (DataRow row in dt.Rows) { if ((int)row["Species"] != prevSpecies || (int)row["Metagame"] != (int)metagames[prevMeta]) { QueueSearch(pid, SearchSpecial.None, (ushort)(prevSpecies), metagames[prevMeta], country, region); return true; } prevSpecies++; if (prevSpecies > 649) { prevSpecies = 1; prevMeta++; } } } } return false; } public static void QueueSpecial(int pid, SearchSpecial special) { QueueSearch(pid, special, 0x0000, SearchMetagames.None, 0x00, 0x00); } public static void QueueSearch(int pid, SearchSpecial special, ushort species, SearchMetagames meta, byte country, byte region) { switch (special) { case SearchSpecial.Latest30: Console.WriteLine("Searching for latest 30 videos."); break; case SearchSpecial.TopLinkBattles: Console.WriteLine("Searching for top link battles."); break; case SearchSpecial.TopSubwayBattles: Console.WriteLine("Searching for top Subway battles."); break; default: { Console.Write("Searching for "); if (species != 0xffff) Console.Write("species {0}, ", species); if (meta != SearchMetagames.None) Console.Write("{0}, ", meta); if (country != 0xff) Console.Write("country {0}, ", region); if (region != 0xff) Console.Write("region {0}", region); } break; } byte[] data = new byte[0x15c]; MemoryStream request = new MemoryStream(data); request.Write(new byte[4], 0, 4); // length goes here, see end request.Write(new byte[] { 0xf1, 0x55, 0x00, 0x00 }, 0, 4); // request type, sanity 0000 request.Write(BitConverter.GetBytes(pid), 0, 4); // pid, hopefully this doesn't ban me request.Write(new byte[] { 0x14, 0x02 }, 0, 2); // there is some random bytes contained in this sometimes. Could be trainer profile // related, I don't know... request.Write(new byte[] { 0x5a, 0x6f, 0x66, 0x78, 0x74, 0x5a, 0x5a, 0x48, 0x54, 0x66, 0x73, 0x4f, 0x43, 0x78, 0x6d, 0x39, 0x76, 0x57, 0x6c, 0x65, 0x63, 0x51, 0x4d, 0x35, 0x37, 0x62, 0x49, 0x34, 0x70, 0x39, 0x56, 0x61, 0x47, 0x59, 0x41, 0x4a, 0x31, 0x4b, 0x70, 0x2f, 0x72, 0x74, 0x4f, 0x68, 0x6c, 0x42, 0x6c, 0x74, 0x72, 0x48, 0x79, 0x71, 0x53, 0x33, 0x57, 0x70, 0x41, 0x34, 0x34, 0x50, 0x38, 0x73, 0x63, 0x65, 0x5a, 0x4f, 0x59, 0x63, 0x51, 0x4d, 0x2b, 0x67, 0x4e, 0x41, 0x46, 0x34, 0x2b, 0x37, 0x6d, 0x32, 0x74, 0x79, 0x78, 0x4b, 0x59, 0x77, 0x3d, 0x3d }, 0, 0x58); request.Write(new byte[0xda], 0, 0xda); request.Write(BitConverter.GetBytes((uint)special), 0, 4); request.Write(BitConverter.GetBytes(species), 0, 2); request.Write(BitConverter.GetBytes((uint)meta), 0, 4); request.WriteByte(country); request.WriteByte(region); request.Write(new byte[] { 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 0, 16); request.Flush(); PutLength(data); if (special != SearchSpecial.Latest30) { using (MySqlConnection db = CreateConnection()) { db.Open(); db.ExecuteNonQuery("INSERT INTO BattleVideoSearchHistory5 (Metagame, Species, " + "Country, Region, Special) VALUES (@metagame, @species, @country, @region, @special)", new MySqlParameter("@metagame", (int)meta), new MySqlParameter("@species", (int)species), new MySqlParameter("@country", (int)country), new MySqlParameter("@region", (int)region), new MySqlParameter("@special", (int)special)); db.Close(); } } byte[] response = Conversation(data); QueueSearchResults(response); } public static void QueueSearchResults(byte[] data) { if (data.Length % 208 != 12) throw new ArgumentException("Search results blob should be 12 bytes + 208 per result."); int count = data.Length / 208; Console.WriteLine("{0} results found.", count); if (count == 0) { // Nothing found. Sleep as to not spam the server with lots of empty searches Thread.Sleep(1000 * 15); return; } // 12 bytes of header plus 208 bytes per search result. using (MySqlConnection db = CreateConnection()) { db.Open(); for (int x = 0; x < count; x++) { ulong videoId = BitConverter.ToUInt64(data, 208 + x * 208); QueueVideoId(db, videoId); } db.Close(); } } public static void QueueVideoId(MySqlConnection db, ulong id) { String formatted = FormatVideoId(id); using (MySqlTransaction tran = db.BeginTransaction()) { long count = (long)tran.ExecuteScalar("SELECT Count(*) FROM BattleVideoCrawlQueue5 WHERE SerialNumber = @serial_number", new MySqlParameter("@serial_number", id)); if (count > 0) { tran.Rollback(); Console.WriteLine("Skipped video {0}. Already present in database.", formatted); return; } tran.ExecuteNonQuery("INSERT INTO BattleVideoCrawlQueue5 (SerialNumber, `Timestamp`) VALUES (@serial_number, NOW())", new MySqlParameter("@serial_number", id)); tran.Commit(); Console.WriteLine("Queued video {0}.", formatted); } } public static ulong DequeueVideo(MySqlConnection db) { using (MySqlTransaction tran = db.BeginTransaction()) { object o = tran.ExecuteScalar("SELECT SerialNumber FROM BattleVideoCrawlQueue5 WHERE Complete = 0 ORDER BY `Timestamp` LIMIT 1"); if (o == null || o == DBNull.Value) { tran.Rollback(); return 0; } ulong id = (ulong)o; tran.ExecuteNonQuery("UPDATE BattleVideoCrawlQueue5 SET Complete = 1 WHERE SerialNumber = @serial_number", new MySqlParameter("@serial_number", id)); tran.Commit(); return id; } } public static void PutLength(byte[] data) { // places the actual length in the first 4 bytes. byte[] length = BitConverter.GetBytes(data.Length); Array.Copy(length, data, 4); } public static byte[] Conversation(byte[] request) { MemoryStream response = new MemoryStream(); using (TcpClient client = new TcpClient("pkgdsprod.nintendo.co.jp", 12401)) { SslStream sslClient = new SslStream(client.GetStream(), false, delegate(object s, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors) { return true; }); sslClient.AuthenticateAsClient("pkgdsprod.nintendo.co.jp"); sslClient.Write(request, 0, request.Length); sslClient.CopyTo(response); sslClient.Close(); } response.Flush(); byte[] dataResponse = response.ToArray(); int length = BitConverter.ToInt32(dataResponse, 0); AssertHelper.Equals(length, dataResponse.Length); return dataResponse; } public static MySqlConnection CreateConnection() { return new MySqlConnection(ConfigurationManager.ConnectionStrings["pkmnFoundationsConnectionString"].ConnectionString); } public static void LogError(Exception ex) { Console.WriteLine(ex.Message); Console.WriteLine(ex.StackTrace); } public enum SearchSpecial : uint { None = 0x00000000, // if special, all other fields are 0 Latest30 = 0x00000001, // 01 00 00 00 00 00 00 00 00 00 00 00 TopLinkBattles = 0x00000003, // 03 00 00 00 00 00 00 00 00 00 00 00 TopSubwayBattles = 0x00000002, // 02 00 00 00 00 00 00 00 00 00 00 00 } public enum SearchMetagames : uint { None = 0x00000000, // Spcial (4), Species (2), Meta (4), Country (1), Region (1) ColosseumSingleNoLauncher = 0x00bf0018, // 00 00 00 00 ff ff 18 00 bf 00 ff ff ColosseumSingleLauncher = 0x00bf0098, // 00 00 00 00 ff ff 98 00 bf 00 ff ff ColosseumDoubleNoLauncher = 0x00bf0019, // 00 00 00 00 ff ff 19 00 bf 00 ff ff ColosseumDoubleLauncher = 0x00bf0099, // 00 00 00 00 ff ff 99 00 bf 00 ff ff ColosseumTripleNoLauncher = 0x00bf001a, // 00 00 00 00 ff ff 1a 00 bf 00 ff ff ColosseumTripleLauncher = 0x00bf009a, // 00 00 00 00 ff ff 9a 00 bf 00 ff ff ColosseumRotationNoLauncher = 0x00bf001b, // 00 00 00 00 ff ff 1b 00 bf 00 ff ff ColosseumRotationLauncher = 0x00bf009b, // 00 00 00 00 ff ff 9b 00 bf 00 ff ff ColosseumMultiNoLauncher = 0x00bf001c, // 00 00 00 00 ff ff 1c 00 bf 00 ff ff ColosseumMultiLauncher = 0x00bf009c, // 00 00 00 00 ff ff 9c 00 bf 00 ff ff BattleSubwaySingle = 0x003f0000, // 00 00 00 00 ff ff 00 00 3f 00 ff ff BattleSubwayDouble = 0x003f0001, // 00 00 00 00 ff ff 01 00 3f 00 ff ff BattleSubwayMulti = 0x003f0004, // 00 00 00 00 ff ff 04 00 3f 00 ff ff RandomMatchupSingle = 0x003f0028, // 00 00 00 00 ff ff 28 00 3f 00 ff ff RandomMatchupDouble = 0x003f0029, // 00 00 00 00 ff ff 29 00 3f 00 ff ff RandomMatchupTriple = 0x003f002a, // 00 00 00 00 ff ff 2a 00 3f 00 ff ff RandomMatchupRotation = 0x003f002b, // 00 00 00 00 ff ff 2b 00 3f 00 ff ff RandomMatchupLauncher = 0x00bf00aa, // 00 00 00 00 ff ff aa 00 bf 00 ff ff BattleCompetition = 0x00380038, // 00 00 00 00 ff ff 38 00 38 00 ff ff } } }