Initial implementation of Battle Video crawler

bvCrawler4/App.config

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<configuration>
+  <connectionStrings>
+    <add name="pkmnFoundationsConnectionString"
+         connectionString="Server=10.211.55.2;Database=gts;User ID=gts;Password=gts;Pooling=true;"
+         providerName="MySql.Data.MySqlClient" />
+  </connectionStrings>
+</configuration>

bvCrawler4/Program.cs

@@ -0,0 +1,264 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using System.Net.Sockets;
+using System.Threading;
+using MySql.Data.MySqlClient;
+using System.Configuration;
+using PkmnFoundations.Data;
+using PkmnFoundations.Support;
+
+namespace bvCrawler4
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            using (FileStream s = File.Open("Box Upload Xor Pad.bin", FileMode.Open))
+            {
+                s.Read(PAD, 0, PAD.Length);
+                s.Close();
+            }
+
+            Console.WriteLine("Pokémon Plat/HG/SS Battle Video Crawler by mm201");
+            int pid = 207823279; // Platinum Hikari
+            Directory.CreateDirectory("videos");
+            DateTime last_top30 = DateTime.MinValue;
+
+            while (true)
+            {
+                ulong videoId;
+                try
+                {
+                    using (MySqlConnection db = CreateConnection())
+                    {
+                        db.Open();
+                        videoId = DequeueVideo(db);
+                        db.Close();
+                    }
+                }
+                catch (Exception ex)
+                {
+                    // haven't touched the server, sleep short
+                    LogError(ex);
+                    Thread.Sleep(1000 * 1);
+                    continue;
+                }
+
+                if (videoId == 0)
+                {
+                    try
+                    {
+                        if (last_top30 < DateTime.Now.AddHours(-1))
+                        {
+                            last_top30 = DateTime.Now;
+                            QueueTop30(pid);
+                            continue;
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        LogError(ex);
+                        Thread.Sleep(1000 * 60);
+                        continue;
+                    }
+
+                    Console.WriteLine("Nothing to do. Idling 1 minute.");
+                    Thread.Sleep(1000 * 60);
+                    continue;
+                }
+
+                byte[] data;
+                String formatted = FormatVideoId(videoId);
+                try
+                {
+                    data = GetBattleVideo(pid, videoId);
+
+                    String filename = String.Format("videos\\{0}.bin", formatted);
+                    using (FileStream file = File.Create(filename))
+                    {
+                        file.Write(data, 0, data.Length);
+                        file.Close();
+                    }
+                }
+                catch (Exception ex)
+                {
+                    LogError(ex);
+                    Thread.Sleep(1000 * 60);
+                    continue;
+                }
+
+                Console.WriteLine("Successfully saved battle video {0}.", formatted);
+                Thread.Sleep(1000 * 60);
+            }
+        }
+
+        private static byte[] PAD = new byte[256];
+
+        public static String FormatVideoId(ulong videoId)
+        {
+            String number = videoId.ToString("D12");
+            String[] split = new String[3];
+            split[0] = number.Substring(0, number.Length - 10);
+            split[1] = number.Substring(number.Length - 10, 5);
+            split[2] = number.Substring(number.Length - 5, 5);
+            return String.Join("-", split);
+        }
+
+        public static byte[] GetBattleVideo(int pid, ulong videoId)
+        {
+            String formatted = FormatVideoId(videoId);
+            Console.WriteLine("Attempting to retrieve battle video {0} from server.", formatted);
+
+            byte[] data = new byte[0x14c];
+            MemoryStream request = new MemoryStream(data);
+            request.Write(new byte[4], 0, 4); // length goes here, see end
+            request.Write(new byte[] { 0xda, 0xae, 0x00, 0x00 }, 0, 4); // request type, sanity 0000
+            request.Write(BitConverter.GetBytes(pid), 0, 4); // pid, hopefully this doesn't ban me
+            request.Write(new byte[] { 0x07, 0x02 }, 0, 2);
+            // there is some random bytes contained in this sometimes. Could be trainer profile
+            // related, I don't know...
+            request.Write(new byte[0x132], 0, 0x132);
+
+            request.Write(BitConverter.GetBytes(videoId), 0, 8);
+            request.Write(new byte[] { 0x40, 0x01, 0x00, 0x00 }, 0, 4);
+            request.Flush();
+            Encrypt(data, 6, 0xba);
+            PutLength(data);
+
+            byte[] response = Conversation(data);
+            Console.WriteLine("Successfully retrieved {0} byte response for battle video {1}.", response.Length, formatted);
+            return response;
+        }
+
+        public static void QueueTop30(int pid)
+        {
+            Console.WriteLine("Searching for latest 30 videos.");
+
+            byte[] data = new byte[0x15c];
+            MemoryStream request = new MemoryStream(data);
+            request.Write(new byte[4], 0, 4); // length goes here, see end
+            request.Write(new byte[] { 0xd9, 0xc4, 0x00, 0x00 }, 0, 4); // request type, sanity 0000
+            request.Write(BitConverter.GetBytes(pid), 0, 4); // pid, hopefully this doesn't ban me
+            request.Write(new byte[] { 0x0c, 0x02 }, 0, 2);
+            // there is some random bytes contained in this sometimes. Could be trainer profile
+            // related, I don't know...
+            request.Write(new byte[0x132], 0, 0x132);
+            request.Write(new byte[] {
+                0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 
+                0xff, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x00, 0xdc, 0xf6, 0x1b, 0x02,
+                0x0c, 0x03, 0x00, 0x00
+            }, 0, 28);
+            request.Flush();
+            Encrypt(data, 6, 0xc9);
+            PutLength(data);
+
+            byte[] response = Conversation(data);
+            Encrypt(response, 6, 0x99);
+            AssertHelper.Assert(response[6] == 0x00);
+            AssertHelper.Assert(response[7] == 0x00); // saaaaanity
+            QueueSearchResults(response);
+        }
+
+        public static void QueueSearchResults(byte[] data)
+        {
+            if (data.Length != 7212) throw new NotSupportedException("Search results blob must be 7212 bytes. (30 results)");
+            // 12 bytes of header plus 240 bytes per search result.
+            using (MySqlConnection db = CreateConnection())
+            {
+                db.Open();
+                for (int x = 0; x < 30; x++)
+                {
+                    ulong videoId = BitConverter.ToUInt64(data, 16 + x * 240);
+                    QueueVideoId(db, videoId);
+                }
+                db.Close();
+            }
+        }
+
+        public static void QueueVideoId(MySqlConnection db, ulong id)
+        {
+            String formatted = FormatVideoId(id);
+            using (MySqlTransaction tran = db.BeginTransaction())
+            {
+                long count = (long)tran.ExecuteScalar("SELECT Count(*) FROM BattleVideoCrawlQueue WHERE SerialNumber = @serial_number", new MySqlParameter("@serial_number", id));
+                if (count > 0)
+                {
+                    tran.Rollback();
+                    Console.WriteLine("Skipped video {0}. Already present.", formatted);
+                    return;
+                }
+                tran.ExecuteNonQuery("INSERT INTO BattleVideoCrawlQueue (SerialNumber, `Timestamp`) VALUES (@serial_number, NOW())", new MySqlParameter("@serial_number", id));
+                tran.Commit();
+                Console.WriteLine("Queued video {0}.", formatted);
+            }
+        }
+
+        public static ulong DequeueVideo(MySqlConnection db)
+        {
+            using (MySqlTransaction tran = db.BeginTransaction())
+            {
+                object o = tran.ExecuteScalar("SELECT SerialNumber FROM BattleVideoCrawlQueue WHERE Complete = 0 ORDER BY `Timestamp` LIMIT 1");
+                if (o == null || o == DBNull.Value)
+                {
+                    tran.Rollback();
+                    return 0;
+                }
+                ulong id = (ulong)o;
+                tran.ExecuteNonQuery("UPDATE BattleVideoCrawlQueue SET Complete = 1 WHERE SerialNumber = @serial_number", new MySqlParameter("@serial_number", id));
+                tran.Commit();
+                return id;
+            }
+        }
+
+        public static void PutLength(byte[] data)
+        {
+            // places the actual length in the first 4 bytes.
+            byte[] length = BitConverter.GetBytes(data.Length);
+            Array.Copy(length, data, 4);
+        }
+
+        public static byte[] Conversation(byte[] request)
+        {
+            MemoryStream response = new MemoryStream();
+
+            using (TcpClient client = new TcpClient("pkgdsprod.nintendo.co.jp", 12400))
+            {
+                NetworkStream s = client.GetStream();
+                s.Write(request, 0, request.Length);
+                s.CopyTo(response);
+                s.Close();
+            }
+            response.Flush();
+            byte[] dataResponse = response.ToArray();
+
+            int length = BitConverter.ToInt32(dataResponse, 0);
+            AssertHelper.Equals(length, dataResponse.Length);
+
+            return dataResponse;
+        }
+
+        public static void Encrypt(byte[] data, int startOffset, int padOffset)
+        {
+            // encrypt and decrypt are the same operation...
+            for (int x = startOffset; x < data.Length; x++)
+            {
+                data[x] ^= PAD[(x + padOffset) % 256];
+            }
+        }
+
+        public static MySqlConnection CreateConnection()
+        {
+            return new MySqlConnection(ConfigurationManager.ConnectionStrings["pkmnFoundationsConnectionString"].ConnectionString);
+        }
+
+        public static void LogError(Exception ex)
+        {
+            Console.WriteLine(ex.Message);
+            Console.WriteLine(ex.StackTrace);
+        }
+    }
+}

bvCrawler4/Properties/AssemblyInfo.cs

@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("bvCrawler4")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("bvCrawler4")]
+[assembly: AssemblyCopyright("Copyright ©  2014")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("01abd719-4ed0-4a30-9f67-663f7cfe03e2")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]

bvCrawler4/bvCrawler4.csproj

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">x86</Platform>
+    <ProductVersion>8.0.30703</ProductVersion>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{04E4EB3B-166B-4D8B-86AA-2178F2A927D5}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>bvCrawler4</RootNamespace>
+    <AssemblyName>bvCrawler4</AssemblyName>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+    <TargetFrameworkProfile>Client</TargetFrameworkProfile>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
+    <PlatformTarget>x86</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' ">
+    <PlatformTarget>x86</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="mysql.data">
+      <HintPath>..\library\lib\mysql.data.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.configuration" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="App.config" />
+    <None Include="Box Upload Xor Pad.bin">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\library\Library.csproj">
+      <Project>{408EFC7E-C6B0-4160-8628-2679E34385CE}</Project>
+      <Name>Library</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>

library/Support/AssertHelper.cs

@@ -43,7 +43,7 @@ namespace PkmnFoundations.Support
 
         public static void Equals<T>(T first, T second) where T : IEquatable<T>
         {
-            Assert((IEquatable<T>)first == (IEquatable<T>)second, "Values are not equal.");
+            Assert(((IEquatable<T>)first).Equals((IEquatable<T>)second), "Values are not equal.");
         }
     }
 }

pkmnFoundations.sln

@@ -9,6 +9,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VeekunImport", "VeekunImpor
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "gts", "gts\gts.csproj", "{2EEAF2A8-68B7-4DB5-8818-36285D5CF9B3}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "bvCrawler4", "bvCrawler4\bvCrawler4.csproj", "{04E4EB3B-166B-4D8B-86AA-2178F2A927D5}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -59,6 +61,16 @@ Global
 		{2EEAF2A8-68B7-4DB5-8818-36285D5CF9B3}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{2EEAF2A8-68B7-4DB5-8818-36285D5CF9B3}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{2EEAF2A8-68B7-4DB5-8818-36285D5CF9B3}.Release|x86.ActiveCfg = Release|Any CPU
+		{04E4EB3B-166B-4D8B-86AA-2178F2A927D5}.Debug|Any CPU.ActiveCfg = Debug|x86
+		{04E4EB3B-166B-4D8B-86AA-2178F2A927D5}.Debug|Mixed Platforms.ActiveCfg = Debug|x86
+		{04E4EB3B-166B-4D8B-86AA-2178F2A927D5}.Debug|Mixed Platforms.Build.0 = Debug|x86
+		{04E4EB3B-166B-4D8B-86AA-2178F2A927D5}.Debug|x86.ActiveCfg = Debug|x86
+		{04E4EB3B-166B-4D8B-86AA-2178F2A927D5}.Debug|x86.Build.0 = Debug|x86
+		{04E4EB3B-166B-4D8B-86AA-2178F2A927D5}.Release|Any CPU.ActiveCfg = Release|x86
+		{04E4EB3B-166B-4D8B-86AA-2178F2A927D5}.Release|Mixed Platforms.ActiveCfg = Release|x86
+		{04E4EB3B-166B-4D8B-86AA-2178F2A927D5}.Release|Mixed Platforms.Build.0 = Release|x86
+		{04E4EB3B-166B-4D8B-86AA-2178F2A927D5}.Release|x86.ActiveCfg = Release|x86
+		{04E4EB3B-166B-4D8B-86AA-2178F2A927D5}.Release|x86.Build.0 = Release|x86
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE