Server browser, natneg

Finally have friend games working. Matchmaking games do not work yet,
though. There are still a number of bugs to be worked out, but it's
possible to complete a few rounds of Tetris DS successfully.

gamespy/gs_server_database.py

@@ -0,0 +1,302 @@
+# Master server list server
+#
+# Basic idea:
+# The server listing does not need to be persistent, and it must be easily searchable for any unknown parameters.
+# So instead of using a SQL database, I've opted to create a server list database server which communicates between the
+# server browser and the qr server. The server list database will be stored in dictionaries as to allow dynamic columns
+# that can be easily searched. The main reason for this configuration is because it cannot be guaranteed what data
+# a game's server will required. For example, in addition to the common fields such as publicip, numplayers, dwc_pid, etc,
+# Lost Magic also uses fields such as LMname, LMsecN, LMrating, LMbtmode, and LMversion.
+#
+# It would be possible to create game-specific databases but this would be more of a hassle and less universal. It would
+# also be possible pickle a dictionary containing all of the fields and store it in a SQL database instead, but that
+# would require unpickling every server each time you want to match search queries which would cause overhead if there
+# are a lot of running servers. One trade off here is that we'll be using more memory by storing each server as a
+# dictionary in the memory instead of storing it in a SQL database.
+#
+# qr_server and server_browser both will act as clients to gs_server_database.
+# qr_server will send an add and/or delete to add or remove servers from the server list.
+# server_browser will send a request with the game name followed by optional search parameters to get a list of servers.
+
+from multiprocessing.managers import BaseManager
+from multiprocessing import freeze_support
+
+server_list = {}
+
+class TokenType:
+    UNKNOWN = 0
+    FIELD = 1
+    STRING = 2
+    NUMBER = 3
+    TOKEN = 4
+
+def get_token(filter, i):
+    # Complex example from Dungeon Explorer: Warriors of Ancient Arts
+    # dwc_mver = 3 and dwc_pid != 474890913 and maxplayers = 2 and numplayers < 2 and dwc_mtype = 0 and dwc_mresv != dwc_pid and (MatchType='english')
+    #
+    # Digging into a few DS games, and these hardcoded search queries seem to be consistent between them:
+    # %s = %d and %s != %u and maxplayers = %d and numplayers < %d and %s = %d and %s != %s
+    # %s and (%s)
+    # %s = %u
+    #
+    # It does not look like OR commands are (at least by default) supported, so for now they won't be implemented.
+    #
+    # Things that must be implemented:
+    #   - and operator
+    #   - integer comparisons
+    #   - string literals comparisons
+    #   - comparison between two fields
+    #   - comparison operators (<, >, =, !=)
+    #   - if not already available, maybe extend the comparison operators to include <= and >= just to be safe
+    #
+    # Things that won't be supported for now unless required:
+    #   - or operator
+    #   - grouping of operators, e.g.: (x or y) and (y or z)
+
+    start = i
+    special_chars = "_"
+
+    token_type = TokenType.UNKNOWN
+
+    # Skip whitespace
+    while i < len(filter) and filter[i].isspace():
+        i += 1
+        start += 1
+
+    if i < len(filter):
+        if filter[i] == "(" or filter[i] == ")":
+            i += 1
+            token_type = TokenType.TOKEN
+
+        elif filter[i] == "=":
+            i += 1
+            token_type = TokenType.TOKEN
+
+        elif filter[i] == ">" or filter[i] == "<":
+            i += 1
+            token_type = TokenType.TOKEN
+
+            if i + 1 < len(filter) and filter[i+1] == "=":
+                # >= or <=
+                i += 1
+
+        elif i + 1 < len(filter) and filter[i] == "!" and filter[i + 1] == "=":
+            i += 2
+            token_type = TokenType.TOKEN
+
+        elif filter[i] == "'":
+            token_type = TokenType.STRING
+
+            i += 1 # Skip quotation mark
+            while i < len(filter) and filter[i] != "'":
+                i += 1
+
+            if i < len(filter) and filter[i] == "'":
+                i += 1 # Skip quotation mark
+
+        elif filter[i] == "\"":
+            # I don't know if it's in the spec or not, but I added "" string literals as well just in case.
+            token_type = TokenType.STRING
+
+            i += 1 # Skip quotation mark
+            while i < len(filter) and filter[i] != "\"":
+                i += 1
+
+            if i < len(filter) and filter[i] == "\"":
+                i += 1 # Skip quotation mark
+
+        elif filter[i].isalnum() or filter[i] in special_chars:
+            # Get whole numbers or words
+            if filter[i].isdigit():
+                token_type = TokenType.NUMBER
+            if filter[i].isalpha():
+                token_type = TokenType.FIELD
+
+            while i < len(filter) and (filter[i].isalnum() or filter[i] in special_chars) and filter[i] not in "!=>< ":
+                i += 1
+
+    if token_type == TokenType.STRING:
+        token = filter[start + 1:i - 1]
+    else:
+        token = filter[start:i]
+
+    return token, i, token_type
+
+def match(filter, i, search):
+    start = i
+    found_match = False
+
+    # Get the next token
+    token, i, _ = get_token(filter, i)
+
+    # If the token isn't the same as what we're searching for, don't move forward.
+    if token != search:
+        i = start
+    else:
+        found_match = True
+
+    return found_match, i
+
+def parse_filter(filter):
+    filters = []
+
+    i = 0
+    found_match = True
+    # Continue while there's a connecting "and".
+    while found_match and i < len(filter):
+        found_match_bracket, i = match(filter, i, "(")
+
+        left, i, _ = get_token(filter, i)
+        if i >= len(filter):
+            break
+
+        op, i, _ = get_token(filter, i)
+        if i >= len(filter):
+            break
+
+        right, i, token_type = get_token(filter, i)
+
+        if found_match_bracket:
+            found_closing_match_bracket, i = match(filter, i, ")")
+
+        found_match, i = match(filter, i, "and")
+
+        filters.append({'op': op, 'left': left, 'right': right, 'type': token_type})
+
+    return filters
+
+def find_servers(gameid, filter, fields, max_count):
+    servers = []
+    if gameid in server_list:
+        filters = parse_filter(filter)
+
+        if max_count <= 0:
+            max_count = 1
+
+        # Generate a list of servers that match the given criteria.
+        for server in server_list[gameid]:
+            if len(servers) > max_count and max_count != -1:
+                break
+
+            matched_filters = 0
+
+            for key in filters:
+                if key['left'] in server:
+                    # Found key, perform actual comparison
+                    right = key['right']
+                    type = key['type']
+
+                    # Only assume that a field will ever reference another field once, so nested references are not supported.
+                    if type == TokenType.FIELD and key['right'] in server:
+                        right = server[key['right']]
+
+                        # Reuse the get_token function to update the new token type of the field in the database.
+                        # Strings will return as fields but the distinction does not matter for the comparisons.
+                        _, _, type = get_token(right, 0)
+
+                    if key['op'] == "=" and server[key['left']] == right:
+                        matched_filters += 1
+                    elif key['op'] == "!=" and server[key['left']] != right:
+                        matched_filters += 1
+                    elif type == TokenType.NUMBER:
+                        # Only perform greater than/less than comparisons on integer types.
+                        if key['op'] == ">" and server[key['left']] > right:
+                            matched_filters += 1
+                        elif key['op'] == ">=" and server[key['left']] >= right:
+                            matched_filters += 1
+                        elif key['op'] == "<" and server[key['left']] < right:
+                            matched_filters += 1
+                        elif key['op'] == "<=" and server[key['left']] <= right:
+                            matched_filters += 1
+                else:
+                    break
+
+            if matched_filters == len(filters):
+                # Create a result with only the fields requested
+                result = {}
+
+                if 'localip0' in server:
+                    # localip1, localip2, ... are possible, but are they ever used?
+                    # Small chance this might cause an issue later.
+                    result['localip0'] = server['localip0']
+
+                if 'localport' in server:
+                    result['localport'] = server['localport']
+
+                if 'localport' in server:
+                    result['localport'] = server['localport']
+
+                if 'natneg' in server:
+                    result['natneg'] = server['natneg']
+
+                if 'publicip' in server:
+                    result['publicip'] = server['publicip']
+
+                if 'publicport' in server:
+                    result['publicport'] = server['publicport']
+
+                if '__session__' in server:
+                    result['__session__'] = server['__session__']
+
+                requested = {}
+                for field in fields:
+                    if not field in result:
+                        if field in server:
+                            requested[field] = server[field]
+                        else:
+                            # Return a dummy value. What's the normal behavior of the real server in this case?
+                            requested[field] = ""
+
+
+                result['requested'] = requested
+                servers.append(result)
+
+    return servers
+
+def update_server_list(gameid, session, value):
+    # Make sure the user isn't hosting multiple servers or there isn't some left over server information that
+    # never got handled properly (game crashed, etc).
+    delete_server(gameid, session)
+
+    # If the game doesn't exist already, create a new list.
+    if not gameid in server_list:
+        server_list[gameid] = []
+
+    # Add new server
+    value['__session__'] = session
+    print "Added %s to the server list for %s" % (gameid, value)
+    server_list[gameid].append(value)
+
+def delete_server(gameid, session):
+    if not gameid in server_list:
+        # Nothing to do if no servers for that game even exist.
+        return
+
+    # Remove all servers hosted by the given session id.
+    server_list[gameid] = [x for x in server_list[gameid] if x['__session__'] != session]
+
+
+class GamespyServerDatabase(BaseManager):
+    pass
+
+def start_server():
+    address = ("127.0.0.1", 27500)
+    password = ""
+
+    #server_list["tetrisds"] = []
+    #server_list["tetrisds"].append({'__session__': 0, 'key': "helloworld", 'value': "Hello, world!", 'extra': "Test"})
+
+    GamespyServerDatabase.register("get_server_list", callable=lambda:server_list)
+    GamespyServerDatabase.register("find_servers", callable=find_servers)
+    GamespyServerDatabase.register("update_server_list", callable=update_server_list)
+    GamespyServerDatabase.register("delete_server", callable=delete_server)
+
+    print "Started server on %s:%d..." % (address[0], address[1])
+
+    manager = GamespyServerDatabase(address = address, authkey = password)
+    server = manager.get_server()
+    server.serve_forever()
+
+if __name__ == '__main__':
+    freeze_support()
+    start_server()

gamespy/gs_utility.py

@@ -1,7 +1,22 @@
 import base64
 import hashlib
+import time
 import other.utils as utils
 
+
+def generate_secret_keys(filename="gslist.cfg"):
+    key_file = open(filename)
+
+    secret_key_list = {}
+    for line in key_file.readlines():
+        #name = line[:54].strip() # Probably won't do anything with the name for now.
+        id = line[54:54+19].strip()
+        key = line[54+19:].strip()
+
+        secret_key_list[id] = key
+
+    return secret_key_list
+
 # GameSpy uses a slightly modified version of base64 which replaces +/= with []_
 def base64_encode(input):
     output = base64.b64encode(input).replace('+', '[').replace('/', ']').replace('=', '_')
@@ -119,4 +134,252 @@ def get_friendcode_from_profileid(profileid, gameid):
 def get_profileid_from_friendcode(friendcode):
     # Get the lower 32 bits as the profile id
     profileid = friendcode & 0xffffffff
-    return profileid
+    return profileid
+
+# Code from Luigi Auriemma's enctypex_decoder.c
+# It's kind of sloppy in parts, but it works. Unless there's some issues then it'll probably not change any longer.
+class EncTypeX:
+    def __init__(self):
+        return
+
+    def decrypt(self, key, validate, data):
+        if not key or not validate or not data:
+            return None
+
+        encxkey = bytearray([0] * 261)
+        data = self.init(encxkey, key, validate, data)
+        self.func6(encxkey, data, len(data))
+
+        return data
+
+    def encrypt(self, key, validate, data):
+        if not key or not validate or not data:
+            return None
+
+        # Convert data from strings to byte arrays before use or else it'll raise an error
+        key = bytearray(key)
+        validate = bytearray(validate)
+
+        # Add room for the header
+        tmp_len = 20
+        data = bytearray(tmp_len) + data
+
+        keylen = len(key)
+        vallen = len(validate)
+        rnd = ~int(time.time())
+
+        for i in range(tmp_len):
+            rnd = (rnd * 0x343FD) + 0x269EC3
+            data[i] = (rnd ^ key[i % keylen] ^ validate[i % vallen]) & 0xff
+
+        header_len = 7
+        data[0] = (header_len - 2) ^ 0xec
+        data[1] = 0x00
+        data[2] = 0x00
+        data[header_len - 1] = (tmp_len - header_len) ^ 0xea
+
+        header = data[:tmp_len]  # The header of the data gets chopped off in init(), so save it
+        encxkey = bytearray([0] * 261)
+        data = self.init(encxkey, key, validate, data)
+        self.func6e(encxkey, data, len(data))
+
+        # Reappend header that we saved earlier before returning to make the complete buffer
+        return header + data
+
+
+    def init(self, encxkey, key, validate, data):
+        data_len = len(data)
+
+        if data_len < 1:
+            return None
+
+        header_len = (data[0] ^ 0xec) + 2
+        if data_len < header_len:
+            return None
+
+        data_start = (data[header_len - 1] ^ 0xea)
+        if data_len < (header_len + data_start):
+            return None
+
+        data = self.enctypex_funcx(encxkey, bytearray(key), bytearray(validate), data[header_len:], data_start)
+        return data[data_start:]
+
+
+    def enctypex_funcx(self, encxkey, key, validate, data, datalen):
+        keylen = len(key)
+
+        for i in range(datalen):
+            validate[(key[i % keylen] * i) & 7] ^= validate[i & 7] ^ data[i]
+
+        self.func4(encxkey, validate, 8)
+        return data
+
+    def func4(self, encxkey, id, idlen):
+        if idlen < 1:
+            return
+
+        for i in range(256):
+            encxkey[i] = i
+
+        n1 = 0
+        n2 = 0
+        for i in range(255,-1,-1):
+            t1, n1, n2 = self.func5(encxkey, i, id, idlen, n1, n2)
+            t2 = encxkey[i]
+            encxkey[i] = encxkey[t1]
+            encxkey[t1] = t2
+
+        encxkey[256] = encxkey[1]
+        encxkey[257] = encxkey[3]
+        encxkey[258] = encxkey[5]
+        encxkey[259] = encxkey[7]
+        encxkey[260] = encxkey[n1 & 0xff]
+
+    def func5(self, encxkey, cnt, id, idlen, n1, n2):
+        if cnt == 0:
+            return 0, n1, n2
+
+        mask = 1
+        doLoop = True
+        if cnt > 1:
+            while doLoop:
+                mask = (mask << 1) + 1
+                doLoop = mask < cnt
+
+        i = 0
+        tmp = 0
+        doLoop = True
+        while doLoop:
+            n1 = encxkey[n1 & 0xff] + id[n2]
+            n2 += 1
+
+            if n2 >= idlen:
+                n2 = 0
+                n1 += idlen
+
+            tmp = n1 & mask
+
+            i += 1
+            if i > 11:
+                tmp %= cnt
+
+            doLoop = tmp > cnt
+
+        return tmp, n1, n2
+
+    def func6(self, encxkey, data, data_len):
+        for i in range(data_len):
+            data[i] = self.func7(encxkey, data[i])
+        return len(data)
+
+    def func7(self, encxkey, d):
+        a = encxkey[256]
+        b = encxkey[257]
+        c = encxkey[a]
+        encxkey[256] = (a + 1) & 0xff
+        encxkey[257] = (b + c) & 0xff
+
+        a = encxkey[260]
+        b = encxkey[257]
+        b = encxkey[b]
+        c = encxkey[a]
+        encxkey[a] = b
+
+        a = encxkey[259]
+        b = encxkey[257]
+        a = encxkey[a]
+        encxkey[b] = a
+
+        a = encxkey[256]
+        b = encxkey[259]
+        a = encxkey[a]
+        encxkey[b] = a
+
+        a = encxkey[256]
+        encxkey[a] = c
+
+        b = encxkey[258]
+        a = encxkey[c]
+        c = encxkey[259]
+        b = (a + b) & 0xff
+        encxkey[258] = b
+
+        a = b
+        c = encxkey[c]
+        b = encxkey[257]
+        b = encxkey[b]
+        a = encxkey[a]
+        c = (b + c) & 0xff
+        b = encxkey[260]
+        b = encxkey[b]
+        c = (b + c) & 0xff
+        b = encxkey[c]
+        c = encxkey[256]
+        c = encxkey[c]
+        a = (a + c) & 0xff
+        c = encxkey[b]
+        b = encxkey[a]
+        encxkey[260] = d
+
+        c ^= b ^ d
+        encxkey[259] = c
+
+        return c
+
+    def func6e(self, encxkey, data, data_len):
+        for i in range(data_len):
+            data[i] = self.func7e(encxkey, data[i])
+        return len(data)
+
+    def func7e(self, encxkey, d):
+        a = encxkey[256]
+        b = encxkey[257]
+        c = encxkey[a]
+        encxkey[256] = (a + 1) & 0xff
+        encxkey[257] = (b + c) & 0xff
+
+        a = encxkey[260]
+        b = encxkey[257]
+        b = encxkey[b]
+        c = encxkey[a]
+        encxkey[a] = b
+
+        a = encxkey[259]
+        b = encxkey[257]
+        a = encxkey[a]
+        encxkey[b] = a
+
+        a = encxkey[256]
+        b = encxkey[259]
+        a = encxkey[a]
+        encxkey[b] = a
+
+        a = encxkey[256]
+        encxkey[a] = c
+
+        b = encxkey[258]
+        a = encxkey[c]
+        c = encxkey[259]
+        b = (a + b) & 0xff
+        encxkey[258] = b
+
+        a = b
+        c = encxkey[c]
+        b = encxkey[257]
+        b = encxkey[b]
+        a = encxkey[a]
+        c = (b + c) & 0xff
+        b = encxkey[260]
+        b = encxkey[b]
+        c = (b + c) & 0xff
+        b = encxkey[c]
+        c = encxkey[256]
+        c = encxkey[c]
+        a = (a + c) & 0xff
+        c = encxkey[b]
+        b = encxkey[a]
+        c ^= b ^ d
+        encxkey[260] = c
+        encxkey[259] = d
+
+        return c

gamespy_server.py

@@ -70,6 +70,9 @@ class PlayerSession(LineReceiver):
                 self.perform_addbuddy(data_parsed)
             elif data_parsed['__cmd__'] == "authadd":
                 self.perform_authadd(data_parsed)
+            else:
+                # Maybe write unknown commands to a separate file later so new data can be collected more easily?
+                utils.print_log("Found unknown command, don't know how to handle '%s'." % data_parsed['__cmd__'])
 
     def perform_login(self, data_parsed):
         authtoken_parsed = gs_utils.parse_authtoken(data_parsed['authtoken'])
@@ -184,12 +187,35 @@ class PlayerSession(LineReceiver):
         self.send_status_to_friends()
 
     def perform_bm(self, data_parsed):
-        dest_profileid = data_parsed['t']
-        dest_msg = data_parsed['msg']
+        if data_parsed['__cmd_val__'] == "1": # Message to/from clients?
+            if "t" in data_parsed:
+                # Send message to the profile id in "t"
+                dest_profileid = int(data_parsed['t'])
+                dest_msg = data_parsed['msg']
+
+                msg_d = []
+                msg_d.append(('__cmd__', "bm"))
+                msg_d.append(('__cmd_val__', "1"))
+                msg_d.append(('f', self.profileid))
+                msg_d.append(('msg', dest_msg))
+                msg = gs_query.create_gamespy_message(msg_d)
+
+                utils.print_log("SENDING TO %s:%s: %s" % (self.sessions[dest_profileid].address.host, self.sessions[dest_profileid].address.port, msg))
+                self.sessions[dest_profileid].transport.write(bytes(msg))
+
 
     def perform_addbuddy(self, data_parsed):
         # Sample: \addbuddy\\sesskey\231601763\newprofileid\476756820\reason\\final\
-        self.db.add_buddy(self.profileid, data_parsed['newprofileid'])
+        buddies = self.db.get_buddy_list(self.profileid)
+
+        buddy_exists = False
+        for buddy in buddies:
+            if buddy['buddyProfileId'] == data_parsed['newprofileid']:
+                buddy_exists = True
+                break
+
+        if not buddy_exists:
+            self.db.add_buddy(self.profileid, data_parsed['newprofileid'])
 
         # In the case that the user is already a buddy:
         # \bm\2\f\217936895\msg\|signed|f259f26d3273f8bda23c7c5e4bd8c5aa\final\

natneg_server.py

@@ -0,0 +1,95 @@
+# Server emulator for *.available.gs.nintendowifi.net and *.master.gs.nintendowifi.net
+# Query and Reporting: http://docs.poweredbygamespy.com/wiki/Query_and_Reporting_Overview
+
+import socket
+import struct
+import gamespy.gs_utility as gs_utils
+import other.utils as utils
+from multiprocessing.managers import BaseManager
+
+session_list = {}
+
+secret_key_list = gs_utils.generate_secret_keys("gslist.cfg")
+
+# Start QR server
+address = ('0.0.0.0', 27901)  # accessible to outside connections (use this if you don't know what you're doing)
+
+s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+s.bind(address)
+
+utils.print_log("Server is now listening on %s:%s..." % (address[0], address[1]))
+
+while 1:
+    recv_data, addr = s.recvfrom(2048)
+
+    print "Connection from %s:%d..." % (addr[0], addr[1])
+
+    # Make sure it's a legal packet
+    if recv_data[0:6] != bytearray([0xfd, 0xfc, 0x1e, 0x66, 0x6a, 0xb2]):
+        continue
+
+    session_id = struct.unpack("<I", recv_data[8:12])[0]
+    session_id_raw = recv_data[8:12]
+
+    # Handle commands
+    if recv_data[7] == '\x00':
+        utils.print_log("Received initialization from %s:%s..." % (addr[0], addr[1]))
+
+        gameid = utils.get_string(recv_data, 0x16)
+        client_id = "%02x" % ord(recv_data[13])
+
+        if gameid not in session_list:
+            session_list[gameid] = {}
+        if session_id not in session_list[gameid]:
+            session_list[gameid][session_id] = {}
+        if client_id not in session_list[gameid][session_id]:
+            session_list[gameid][session_id][client_id] = { 'connected': False, 'addr': '' }
+
+        session_list[gameid][session_id][client_id]['addr'] = addr
+        clients = len(session_list[gameid][session_id])
+        if client_id in session_list[gameid][session_id]:
+            clients -= 1
+
+        if clients > 0:
+            # Someone else is waiting to connect, send message
+            for client in session_list[gameid][session_id]:
+                if session_list[gameid][session_id][client]['connected'] == True:
+                    continue
+
+                output = bytearray(recv_data[0:12])
+                if client == client_id:
+                    output += bytearray(recv_data[15:15+4+2]) # IP Address and port
+                else:
+                    output += bytearray([int(x) for x in addr[0].split('.')])
+                    output += utils.get_bytes_from_short_be(addr[1])
+
+                output += bytearray([0x42, 0x00]) # Unknown, always seems to be \x42\x00
+                output[7] = 0x05
+                s.sendto(output, (session_list[gameid][session_id][client]['addr']))
+
+                print "Sent connection request to %s:%d..." % (session_list[gameid][session_id][client]['addr'][0], session_list[gameid][session_id][client]['addr'][1])
+
+        output = bytearray(recv_data[0:14])
+        output += bytearray([0xff, 0xff, 0x6d, 0x16, 0xb5, 0x7d, 0xea ]) # Checked with Tetris DS, Mario Kart DS, and Metroid Prime Hunters, and this seems to be the standard response to 0x00
+        output[7] = 0x01 # Initialization response
+        s.sendto(output, addr)
+
+    if recv_data[7] == '\x06': # Was able to connect
+        client_id = "%02x" % ord(recv_data[14])
+
+        if gameid not in session_list:
+            pass
+        if session_id not in session_list[gameid]:
+            pass
+        if client_id not in session_list[gameid][session_id]:
+            pass
+
+        session_list[gameid][session_id][client_id]['connected'] = True
+
+
+
+
+
+
+
+

other/utils.py

@@ -1,5 +1,6 @@
 import random
 import time
+import sys
 
 def generate_random_str(len):
     return ''.join(random.choice("abcdefghjiklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") for _ in range(len))
@@ -82,7 +83,10 @@ def get_num_from_bytes(data, idx, bytes, bigEndian = False):
     num = 0
     i = 0
     while i < bytes and i < len(data):
-        num |= (ord(data[i]) << (8 * i))
+        if type(data[i]) is int:
+            num |= (data[i] << (8 * i))
+        else:
+            num |= (ord(data[i]) << (8 * i))
         i += 1
 
     return num
@@ -103,4 +107,52 @@ def get_int_be(data, idx):
 def get_string(data, idx):
     data = data[idx:]
     end = data.index('\0')
-    return data[:end]
+    return str(data[:end])
+
+def get_bytes_from_num(num, size, bigEndian = False):
+    output = bytearray(size)
+
+    i = 0
+    while i < size:
+        output[i] = (num >> 8 * i) & 0xff
+
+        i += 1
+
+    if bigEndian:
+        output = output[::-1]
+
+    return output
+
+def get_bytes_from_short(num):
+    return get_bytes_from_num(num, 2, False)
+
+def get_bytes_from_short_be(num):
+    return get_bytes_from_num(num, 2, True)
+
+def get_bytes_from_int(num):
+    return get_bytes_from_num(num, 4, False)
+
+def get_bytes_from_int_be(num):
+    return get_bytes_from_num(num, 4, True)
+
+def print_hex(data, cols = 16):
+    for i in range(len(data) / cols + 1):
+        c = 0
+        for x in range(cols):
+            if (i * cols + x + 1) > len(data):
+                break
+
+            print "%02x" % data[i * cols + x],
+            c += 1
+
+        c = cols - c
+        sys.stdout.write(" " * (c * 3 + 1))
+        for x in range(cols):
+            if (i * cols + x + 1) > len(data):
+                break
+
+            if data[i * cols + x] < 0x21 or data[i * cols + x] >= 0x7f:
+                sys.stdout.write(".")
+            else:
+                sys.stdout.write("%c" % data[i * cols + x])
+        print ""

qr_server.py

@@ -1,17 +1,41 @@
 # Server emulator for *.available.gs.nintendowifi.net and *.master.gs.nintendowifi.net
 # Query and Reporting: http://docs.poweredbygamespy.com/wiki/Query_and_Reporting_Overview
 
-# TODO: Refactor into a class
-
 import socket
+import struct
 import gamespy.gs_utility as gs_utils
 import other.utils as utils
+from multiprocessing.managers import BaseManager
 
-def get_game_id(data):
-    game_id = data[5: -1]
-    return game_id
+session_list = {}
+class Session(object):
+    def __init__(self, address):
+        self.session = ""
+        self.challenge = ""
+        self.secretkey = "" # Parse gslist.cfg later
+        self.sent_challenge = False
+        self.address = addr
 
-#address = ('127.0.0.1', 27900) # accessible to only the local computer
+
+# Generate a dictionary "secret_key_list" containing the secret game keys associated with their game IDs.
+# The dictionary key will be the game's ID, and the value will be the secret key.
+secret_key_list = gs_utils.generate_secret_keys("gslist.cfg")
+utils.print_log("Generated list of secret game keys...")
+
+# Initialize server list server connection
+class GamespyServerDatabase(BaseManager):
+    pass
+
+GamespyServerDatabase.register("update_server_list")
+GamespyServerDatabase.register("delete_server")
+
+manager_address = ("127.0.0.1", 27500)
+manager_password = ""
+
+server_manager = GamespyServerDatabase(address = manager_address, authkey= manager_password)
+server_manager.connect()
+
+# Start QR server
 address = ('0.0.0.0', 27900)  # accessible to outside connections (use this if you don't know what you're doing)
 
 s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
@@ -19,11 +43,6 @@ s.bind(address)
 
 utils.print_log("Server is now listening on %s:%s..." % (address[0], address[1]))
 
-# Temporarily make these global until everything is made into a client class.
-server_challenge = ""
-secretkey = "JJlSi8" # Parse gslist.cfg later
-sent_challenge = False
-
 while 1:
     recv_data, addr = s.recvfrom(2048)
 
@@ -109,6 +128,13 @@ while 1:
     # Open source version of GameSpy found here: https://github.com/sfcspanky/Openspy-Core/tree/master/qr
     # Use as reference.
 
+    session_id = struct.unpack("<I", recv_data[1:5])[0]
+    session_id_raw = recv_data[1:5]
+    if session_id not in session_list:
+        # Found a new session, add to session list
+        session_list[session_id] = Session(addr)
+
+    # Handle commands
     if recv_data[0] == '\x00': # Query
         utils.print_log("NOT IMPLEMENTED! Received query from %s:%s... %s" % (addr[0], addr[1], recv_data[5:]))
 
@@ -116,20 +142,22 @@ while 1:
         utils.print_log("Received challenge from %s:%s... %s" % (addr[0], addr[1], recv_data[5:]))
 
         # Prepare the challenge sent from the server to be compared
-        challenge = gs_utils.prepare_rc4_base64(secretkey, server_challenge)
+        challenge = gs_utils.prepare_rc4_base64(session_list[session_id].secretkey, session_list[session_id].challenge)
 
         # Compare challenge
         client_challenge = recv_data[5:-1]
         if client_challenge == challenge:
             # Challenge succeeded
-            sent_challenge = True
+            session_list[session_id].sent_challenge = True
 
             # Handle successful challenge stuff here
             packet = bytearray([0xfe, 0xfd, 0x0a]) # Send client registered command
-            packet.extend(recv_data[1:5]) # Get the ID
+            packet.extend(session_id_raw) # Get the session ID
             s.sendto(packet, addr)
             utils.print_log("Sent client registered to %s:%s..." % (addr[0], addr[1]))
 
+            # TODO: Send buddy list to client on successful connection.
+
     elif recv_data[0] == '\x02': # Echo
         utils.print_log("NOT IMPLEMENTED! Received echo from %s:%s... %s" % (addr[0], addr[1], recv_data[5:]))
 
@@ -142,24 +170,41 @@ while 1:
 
         # It may be safe to ignore "unknown" keys because the proper key names get filled in later...
         k = {}
-        for i in range(len(d) / 2):
+        i = 0
+        while i < len(d):
+            print "%s = %s" % (d[i], d[i+1])
             k[d[i]] = d[i+1]
+            i += 2
 
-        # Store k per client
+        if "gamename" in k:
+            session_list[session_id].secretkey = secret_key_list[k['gamename']]
+            #print "Got secret key %s for %s" % (session_list[session_id].secretkey, k['gamename'])
 
-        if sent_challenge == False:
+        if session_list[session_id].sent_challenge == False:
             addr_hex =  ''.join(["%02X" % int(x) for x in addr[0].split('.')])
             port_hex = "%04X" % int(addr[1])
             server_challenge = utils.generate_random_str(8) + addr_hex + port_hex
 
+            session_list[session_id].challenge = server_challenge
+
             packet = bytearray([0xfe, 0xfd, 0x01]) # Send challenge command
-            packet.extend(recv_data[1:5]) # Get the ID
+            packet.extend(session_id_raw) # Get the session ID
             packet.extend(server_challenge)
             packet.extend('\x00')
 
             s.sendto(packet, addr)
             utils.print_log("Sent challenge to %s:%s..." % (addr[0], addr[1]))
 
+        if "statechanged" in k:
+            if k['statechanged'] == "1": # Create server
+                if k['publicport'] != "0" and k['publicip'] != "0" and k['maxplayers'] != "0":
+                    # Some memory could be saved by clearing out any unwanted fields from k before sending.
+                    server_manager.update_server_list(k['gamename'] , session_id, k)
+            elif k['statechanged'] == "2": # Close server
+                server_manager.delete_server(k['gamename'] , session_id)
+                #session_list.pop(session_id)
+
+
     elif recv_data[0] == '\x04': # Add Error
         utils.print_log("NOT IMPLEMENTED! Received add error from %s:%s... %s" % (addr[0], addr[1], recv_data[5:]))
 
@@ -177,9 +222,9 @@ while 1:
 
     elif recv_data[0] == '\x09': # Available
         # Availability check only sent to *.available.gs.nintendowifi.net
-        utils.print_log("Received availability request for '%s' from %s:%s..." % (get_game_id(recv_data), addr[0], addr[1]))
+        utils.print_log("Received availability request for '%s' from %s:%s..." % (recv_data[5: -1], addr[0], addr[1]))
 
-        s.sendto(bytearray([0xfe, 0xfd, recv_data[0], recv_data[1], recv_data[2], recv_data[3], recv_data[4]]), addr)
+        s.sendto(bytearray([0xfe, 0xfd, 0x09, 0x00, 0x00, 0x00, 0x00]), addr)
 
     elif recv_data[0] == '\x0a': # Client Registered
         # Only sent to client, never received?

server_browser.py

@@ -1,46 +1,108 @@
 # I found an open source implemention of this exact server I'm trying to emulate here: https://github.com/sfcspanky/Openspy-Core/blob/master/serverbrowsing/
 # Use as reference later.
 
-# Tetris DS won't let you search for a match unless this server exists, so just create an empty server for now.
+from twisted.internet.protocol import Factory
+from twisted.internet.endpoints import serverFromString
+from twisted.protocols.basic import LineReceiver
+from twisted.internet import reactor
 
 import socket
+import ctypes
+
 import gamespy.gs_utility as gs_utils
 import other.utils as utils
-import time
+
+from multiprocessing.managers import BaseManager
 
 def get_game_id(data):
     game_id = data[5: -1]
     return game_id
 
-#address = ('127.0.0.1', 28910) # accessible to only the local computer
-address = ('0.0.0.0', 28910)  # accessible to outside connections (use this if you don't know what you're doing)
-backlog = 10
-size = 2048
+def get_server_list(game, filter, fields, max_count):
+    results = server_manager.find_servers(game, filter, fields, max_count)
+    return results
 
-s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-s.bind(address)
-s.listen(backlog)
+class ServerListFlags:
+    UNSOLICITED_UDP_FLAG = 1
+    PRIVATE_IP_FLAG = 2
+    CONNECT_NEGOTIATE_FLAG = 4
+    ICMP_IP_FLAG = 8
+    NONSTANDARD_PORT_FLAG = 16
+    NONSTANDARD_PRIVATE_PORT_FLAG = 32
+    HAS_KEYS_FLAG = 64
+    HAS_FULL_RULES_FLAG = 128
 
-utils.print_log("Server is now listening on %s:%s..." % (address[0], address[1]))
+def generate_server_list_data(address, fields, server_info):
+    output = bytearray()
 
-while 1:
-    client, addr = s.accept()
+    # Write the address
+    output += bytearray([int(x) for x in address.host.split('.')])
 
-    utils.print_log("Received connection from %s:%s" % (address[0], address[1]))
+    # Write the port
+    output += utils.get_bytes_from_short_be(address.port)
 
-    receive_data = True
-    while receive_data:
-        utils.print_log("Waiting for data...")
+    if len(server_info) > 0:
+        # Write number of fields that will be returned.
+        key_count = len(server_info['requested'])
+        output += utils.get_bytes_from_short(key_count)
 
-        accept_connection = True
-        data = client.recv(size)
+        if key_count != len(fields):
+            # For some reason we didn't get all of the expected data.
+            print "key_count[%d] != len(fields)[%d]" % (key_count, len(fields))
+            print fields
+            return
 
-        # Didn't get a valid command.
-        # Player disconnected?
-        # Close connection
-        if not data:
-            break
+        flags_buffer = bytearray()
 
+        # Write the fields
+        for field in fields:
+            output += bytearray(field) + '\0\0'
+
+        # Start server loop here instead of including all of the fields and stuff again
+        flags = 0
+        if key_count > 0:
+            flags |= ServerListFlags.HAS_KEYS_FLAG
+
+        if "natneg" in server_info:
+            flags |= ServerListFlags.CONNECT_NEGOTIATE_FLAG
+
+        flags_buffer += utils.get_bytes_from_int(int(server_info['publicip']))
+
+        flags |= ServerListFlags.NONSTANDARD_PORT_FLAG
+        flags_buffer += utils.get_bytes_from_short_be(int(server_info['publicport']))
+
+        if "localip0" in server_info:
+            flags |= ServerListFlags.PRIVATE_IP_FLAG
+            flags_buffer += bytearray([int(x) for x in server_info['localip0'].split('.')])
+
+        if "localport" in server_info:
+            flags |= ServerListFlags.NONSTANDARD_PRIVATE_PORT_FLAG
+            flags_buffer += utils.get_bytes_from_short_be(int(server_info['localport']))
+
+        flags |= ServerListFlags.ICMP_IP_FLAG
+        flags_buffer += bytearray([int(x) for x in "0.0.0.0".split('.')])
+
+        output += bytearray([flags & 0xff])
+        output += flags_buffer
+
+        if (flags & ServerListFlags.HAS_KEYS_FLAG):
+            # Write data for associated fields
+            for field in fields:
+                output += '\xff' + bytearray(server_info['requested'][field]) + '\0'
+
+        output += '\0'
+        output += utils.get_bytes_from_int(-1)
+
+    return output
+
+class Session(LineReceiver):
+    def __init__(self, addr):
+        self.setRawMode() # We're dealing with binary data so set to raw mode
+        self.addr = addr
+        self.forward_to_client = False
+        self.forward_client = ()
+
+    def rawDataReceived(self, data):
         # First 2 bytes are the packet size.
         #
         # Third byte is the command byte.
@@ -53,8 +115,32 @@ while 1:
         #   0x05 - Player search request
         #
         # For Tetris DS, at the very least 0x00 and 0x02 need to be implemented.
+
+        if self.forward_to_client:
+            # Find session id of server
+            # Iterate through the list of servers sent to the client and match by IP and port.
+            # Is there a better way to determine this information?
+            ip = str(ctypes.c_int32(utils.get_int(bytearray([int(x) for x in self.forward_client[0].split('.')]), 0)).value)
+            for server in self.server_list:
+                print "%s %s" % (ip, server['publicip'])
+                if server['publicip'] == ip and server['publicport'] == str(self.forward_client[1]):
+                    print server
+
+                    # Send command to server to get it to connect to natneg
+                    natneg_session = int(utils.generate_random_hex_str(8), 16) # Quick and lazy way to get a random 32bit integer. Replace with something else late.r
+
+                    output = bytearray([0xfe, 0xfd, 0x06])
+                    output += utils.get_bytes_from_int(server['__session__'])
+                    output += bytearray(utils.get_bytes_from_int(natneg_session))
+                    output += bytearray(data)
+
+                    client_s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+                    client_s.sendto(output, self.forward_client)
+                    utils.print_log("Forwarded data to %s:%s..." % (self.forward_client[0], self.forward_client[1]))
+            return
+
         if data[2] == '\x00': # Server list request
-            utils.print_log("Received server list request from %s:%s..." % (addr[0], addr[1]))
+            utils.print_log("Received server list request from %s:%s..." % (self.addr.host, self.addr.port))
 
             # This code is so... not python. The C programmer in me is coming out strong.
             # TODO: Rewrite this section later?
@@ -92,38 +178,82 @@ while 1:
             elif (options & ALTERNATE_SOURCE_IP):
                 source_ip = utils.get_int(data, idx)
 
-            print "%02x %02x %08x" % (list_version, encoding_version, game_version)
-            print "%s" % query_game
+            if '\\' in fields:
+                fields = [x for x in fields.split('\\') if x and not x.isspace()]
+
+            #print "%02x %02x %08x" % (list_version, encoding_version, game_version)
+            #print "%s" % query_game
             print "%s" % game_name
-            print "%s" % challenge
+            #print "%s" % challenge
             print "%s" % filter
             print "%s" % fields
 
-            print "%08x" % options
-            print "%d %08x" % (max_servers, source_ip)
+            #print "%08x" % options
+            #print "%d %08x" % (max_servers, source_ip)
+
+            # Get dictionary from master server list server.
+            self.server_list = get_server_list(query_game, filter, fields, max_servers)._getvalue()
+
+            # Generate encrypted server list and send to client.
+            print self.server_list
+            for server in self.server_list:
+                # Generate binary server list data
+                data = generate_server_list_data(self.addr, fields, server)
+
+                # Encrypt data
+                enc = gs_utils.EncTypeX()
+                data = enc.encrypt(secret_key_list[game_name], challenge, data)
+
+                # Send to client
+                self.transport.write(bytes(data))
+                utils.print_log("Sent server list message to %s:%s..." % (self.addr.host, self.addr.port))
+                break
 
-            # TODO: Handle query
 
         elif data[2] == '\x02': # Send message request
-            dest_addr = '.'.join(["%d" % x for x in addr[3:7]])
-            dest_port = utils.get_short_be(addr, 7) # What's the pythonic way to do this? unpack?
+            dest_addr = '.'.join(["%d" % ord(x) for x in data[3:7]])
+            dest_port = utils.get_short_be(data, 7) # What's the pythonic way to do this? unpack?
             dest = (dest_addr, dest_port)
 
-            # Wait for message data
-            msg_data = client.recv(size)
+            utils.print_log("Received send message request from %s:%s to %s:%d..." % (self.addr.host, self.addr.port, dest_addr, dest_port))
 
-            utils.print_log("Received send message request from %s:%s to %s:%d... %s" % (addr[0], addr[1], dest_addr, dest_port, msg_data))
-
-            # Create new connection to send to other user over UDP.
-            # Move this code somewhere else after testing has been finished.
-            user_s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-            user_s.bind(dest)
-            user_s.sendto(msg_data, dest)
-
-            utils.print_log("Sent message to %s:%d... %s" % (dest_addr, dest_port, msg_data))
+            self.forward_to_client = True
+            self.forward_client = dest
 
         elif data[2] == '\x03': # Keep alive reply
-            utils.print_log("Received keep alive from %s:%s..." % (addr[0], addr[1]))
+            utils.print_log("Received keep alive from %s:%s..." % (self.addr.host, self.addr.port))
 
         else:
-            utils.print_log("Received unknown command (%02x) from %s:%s... %s" % (ord(data[2]), addr[0], addr[1], data))
+            utils.print_log("Received unknown command (%02x) from %s:%s... %s" % (ord(data[2]), self.addr.host, self.addr.port, data))
+
+
+class SessionFactory(Factory):
+    def __init__(self):
+        print "Now listening for connections..."
+
+    def buildProtocol(self, addr):
+        return Session(addr)
+
+
+
+
+
+# Initialize server list server connection
+class GamespyServerDatabase(BaseManager):
+    pass
+
+GamespyServerDatabase.register("get_server_list")
+GamespyServerDatabase.register("modify_server_list")
+GamespyServerDatabase.register("find_servers")
+
+manager_address = ("127.0.0.1", 27500)
+manager_password = ""
+
+server_manager = GamespyServerDatabase(address = manager_address, authkey= manager_password)
+server_manager.connect()
+
+secret_key_list = gs_utils.generate_secret_keys("gslist.cfg")
+
+endpoint = serverFromString(reactor, "tcp:28910")
+conn = endpoint.listen(SessionFactory())
+reactor.run()

www/available.nintendowifi.net/public_html/index.html

@@ -0,0 +1 @@
+test 2

www/nas.nintendowifi.net/public_html/ac

@@ -1,4 +1,6 @@
 <?php
+// Make sure to set "ForceType application/x-httpd-php" somewhere in Apache for this file (vhost configuration works well enough)
+
 // Return the same headers as the real server does for the sake of completeness.
 // They never get checked so they are entirely optional.
 header("NODE: wifiappw3");
@@ -62,7 +64,7 @@ foreach ($_POST as $key => $value)
 		$key == "passwd" ||
 		//$key == "bssid" ||
 		//$key == "apinfo" ||
-		//$key == "gamecd" ||
+		$key == "gamecd" ||
 		//$key == "makercd" ||
 		//$key == "unitcd" ||
 		//$key == "macadr" || 
@@ -90,7 +92,7 @@ foreach ($_GET as $key => $value)
 		$key == "passwd" ||
 		//$key == "bssid" ||
 		//$key == "apinfo" ||
-		//$key == "gamecd" ||
+		$key == "gamecd" ||
 		//$key == "makercd" ||
 		//$key == "unitcd" ||
 		//$key == "macadr" ||