Leahnaya/gfdm-server
1
0
Fork 0
mirror of https://github.com/573dev/gfdm-server.git synced 2026-07-05 18:56:21 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add lz77 implementation. Work on "local" services

Browse Source
This commit is contained in:
573dev 2020-10-14 13:29:19 -05:00
parent 6967dfa4b3
commit 6ff677845d
4 changed files with 441 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
setup.py
View File

@ -13,7 +13,6 @@ REQUIREMENTS = [
"pyopenssl",
"lxml",
"pycryptodome",
"lzss",
"kbinxml",
]

412
v8_server/utils/lz77.py Normal file
View File

@ -0,0 +1,412 @@
# Code directly from:
# https://github.com/ByteFun/bemaniutils/blob/bd467a9b732a25a1c8aba75106dc459fbdff61b0/
# bemani/protocol/lz77.py
from collections import defaultdict
from typing import Generator, List, Mapping, Optional, Set, Tuple
class LzException(Exception):
"""
An exception thrown when we encounter an error with Lz77 encoding/decoding.
"""
class Lz77Decompress:
"""
A class that can decompress an Lz77 stream of data. Notably, this is a different
variant to the Lz77 found in firebeat executables and BIOS. This is used for
over-the-wire compression of XML data, as well as compression inside a decent
amount of file formats found in various Konami games.
"""
RING_LENGTH = 0x1000
FLAG_COPY = 1
FLAG_BACKREF = 0
def __init__(self, data: bytes, backref: Optional[int] = None) -> None:
"""
Initialize the object.
Parameters:
data - Binary blob representing the data to be decompressed.
"""
self.eof: bool = False
self.data: bytes = data
self.read_pos: int = 0
self.left: int = len(self.data)
self.flags: int = 1
self.write_pos: int = 0
self.pending_copy_amount: int = 0
self.pending_copy_pos: int = 0
self.pending_copy_max: int = 0
self.ringlength: int = backref or self.RING_LENGTH
self.ring: bytes = b"\x00" * self.ringlength
def __ring_read(self, copy_pos: int, copy_len: int) -> Generator[bytes, None, None]:
"""
Read the next bytes from the backref ring at the current copy position.
Returns:
a generator which yields bytes.
"""
while copy_len > 0:
if copy_pos + copy_len > self.ringlength:
# Copy first chunk, let subsequent loop handle the next chunks
amount = self.ringlength - copy_pos
else:
# Copy the whole thing out, we have enough space to do so
amount = copy_len
ret = self.ring[copy_pos : (copy_pos + amount)]
self.__ring_write(ret)
yield ret
copy_pos = (copy_pos + amount) % self.ringlength
copy_len -= amount
def __ring_write(self, bytedata: bytes) -> None:
"""
Write bytes into the backref ring.
Parameters:
byte - A byte to be written at the current write offset
"""
while True:
amount = len(bytedata)
if amount == 0:
return
if amount > (self.ringlength - self.write_pos):
amount = self.ringlength - self.write_pos
self.ring = (
self.ring[: self.write_pos]
+ bytedata[:amount]
+ self.ring[(self.write_pos + amount) :]
)
bytedata = bytedata[amount:]
self.write_pos = (self.write_pos + amount) % self.ringlength
def decompress_bytes(self) -> Generator[bytes, None, None]:
"""
Yield the next byte from the decompressed output. If we are
in a backref copy, read the next byte from the backref. If
we aren't, read the next flag to see if we should decode
a byte directly or if we should start another backref read.
If we don't have any flags, exit reporting EOF (None). If
we hit the end of the stream, stop yielding to signify EOF.
In all cases, whatever byte we read should be added back to
the backref buffer.
Returns:
a generator that yields bytes.
"""
while not self.eof:
if self.pending_copy_amount > 0:
# We had a backref that would have copied more data than we had
# available in the ringbuffer (because every read, even a backref, adds
# to the ringbuffer). So, since we read that last time and wrote it to
# the backbuffer we are safe to read again.
amount = min(self.pending_copy_amount, self.pending_copy_max)
yield from self.__ring_read(self.pending_copy_pos, amount)
# We read this many bytes and are about to write them to the ringbuffer,
# so bookkeep that.
self.pending_copy_amount -= amount
self.pending_copy_max = amount
else:
if self.flags == 1:
# Load the next byte for processing
if self.left == 0:
# We have nothing left to read, so skip the
# ringbuffer write below and exit early.
return
else:
self.flags = 0x100 | self.data[self.read_pos]
self.read_pos += 1
self.left -= 1
# Shift the lowest bit out to be retrieved as a flag
flag = self.flags & 1
self.flags >>= 1
if flag == self.FLAG_COPY:
# Figure out how much to pull at once
amount = 1
while self.flags != 1 and (self.flags & 1) == self.FLAG_COPY:
# We would do a copy next time, so pop that flag and just add to
# our read amount
self.flags >>= 1
amount += 1
# Grab chunk right out of the data source
b = self.data[self.read_pos : (self.read_pos + amount)]
self.__ring_write(b)
yield b
self.read_pos += amount
self.left -= amount
elif flag == self.FLAG_BACKREF:
yield from self.__read_backref()
else:
raise Exception("Logic error!")
def __read_backref(self) -> Generator[bytes, None, None]:
"""
Read a backref chunk. Grab the copy length and copy position
from the first two bytes and then read the first byte from
the backref. Sets up variables such that Lz77Decompress.__read()
can finish copying out of the backref on subsequent calls. Should
only be called by Lz77Decompress.__read(). If we discover the end
of stream, we don't generate any bytes and instead set the eof
flag which terminates the main decompression loop above.
Returns:
a generator that yields bytes.
"""
if self.left == 0:
self.eof = True
return
if self.left == 1:
raise LzException("Unexpected EOF mid-backref")
hi = self.data[self.read_pos]
lo = self.data[self.read_pos + 1]
self.read_pos += 2
self.left -= 2
copy_len = lo & 0xF
copy_pos = (hi << 4) | (lo >> 4)
if copy_pos > 0:
copy_len += 3
if copy_len > copy_pos:
# Remember what we have to do left, and the safe
# amount to copy next time (which is our length,
# since we are about to write that many butes to
# the ringbuffer right after reading them).
self.pending_copy_amount = copy_len - copy_pos
self.pending_copy_pos = self.write_pos
self.pending_copy_max = copy_pos
# Only copy the available bytes
copy_len = copy_pos
copy_pos = self.write_pos - copy_pos
while copy_pos < 0:
copy_pos += self.ringlength
copy_pos = copy_pos % self.ringlength
yield from self.__ring_read(copy_pos, copy_len)
else:
self.eof = True
return
class Lz77Compress:
"""
A class that can compress arbitrary binary data using the Lz77 protocol.
Note that this does support overlapped backtracks, so for instance the
string "abcabcabc" will be compressed properly (see unit tests for examples).
Great care has been taken in optimizing this and then we further optimize
by using Cython to build, netting us another 40% speed-up. This is important
because for any given packet we are decompressing and compressing at least
once, and if we use a proxy to direct traffic, possibly a second time.
"""
RING_LENGTH = 0x1000
FLAG_COPY = 1
FLAG_BACKREF = 0
def __init__(self, data: bytes, backref: Optional[int] = None) -> None:
"""
Initialize the object.
Parameters:
data - Binary blob representing the data to be decompressed.
"""
self.data: bytes = data
self.read_pos: int = 0
self.left: int = len(self.data)
self.eof: bool = False
self.bytes_written: int = 0
self.ringlength: int = backref or self.RING_LENGTH
self.locations: Mapping[int, Set[int]] = defaultdict(set)
self.starts: Mapping[bytes, Set[int]] = defaultdict(set)
self.last_start: Tuple[int, int, int] = (0, 0, 0)
def __ring_write(self, bytedata: bytes) -> None:
"""
Write bytes into the backref ring.
Parameters:
byte - A byte to be written at the current write offset
"""
for byte in bytedata:
# Update the start locations hashmap if we're past the beginning
self.last_start = (self.last_start[1], self.last_start[2], byte)
if self.bytes_written >= 2:
self.starts[bytes(self.last_start)].add(self.bytes_written - 2)
# Update the rest of the location hashmaps
self.locations[byte].add(self.bytes_written)
self.bytes_written += 1
def compress_bytes(self) -> Generator[bytes, None, None]:
"""
Given the current stream, go through and assemble the next flag byte
followed by the next chunk of compressed data.
"""
while not self.eof:
if self.left == 0:
# Output a dummy flag and an end of stream marker.
self.eof = True
yield b"\x00\x00\x00"
else:
# Need to assemble and return the next chunk, which is a flag
# byte and then 8 instructions.
flags = 0x0
flagpos = -1
data: List[bytes] = [b"\x00"] * 8
for _ in range(8):
# Track what flag we're generating data for
flagpos += 1
if self.left == 0:
# Output the end of stream marker, set EOF since we've succeeded
# in outputting all flags.
flags |= self.FLAG_BACKREF << flagpos
data[flagpos] = b"\x00\x00"
self.eof = True
break
elif self.left < 3 or self.bytes_written < 3:
# We either don't have enough data written to backref, or we
# don't have enough data in the stream that could be made into
# a backref.
flags |= self.FLAG_COPY << flagpos
chunk = self.data[self.read_pos : (self.read_pos + 1)]
data[flagpos] = chunk
self.__ring_write(chunk)
self.read_pos += 1
self.left -= 1
continue
# Figure out the maximum backref we can attempt to find
backref_amount = min(self.left, 18)
# Iterate over all spots where the first byte equals, and is in
# range.
earliest = max(0, self.bytes_written - self.ringlength)
possible_backref_locations: List[int] = [
absolute_pos
for absolute_pos in self.starts[
self.data[self.read_pos : (self.read_pos + 3)]
]
if absolute_pos >= earliest
]
# Output the data as a copy if we couldn't find a backref
if not possible_backref_locations:
flags |= self.FLAG_COPY << flagpos
chunk = self.data[self.read_pos : (self.read_pos + 1)]
data[flagpos] = chunk
self.__ring_write(chunk)
self.read_pos += 1
self.left -= 1
continue
# Now, find the longest actual backref of our possibilities. We know
# we're going to write at least these three bytes, so append it to
# the output buffer.
start_write_size = self.bytes_written
self.__ring_write(self.data[self.read_pos : (self.read_pos + 3)])
copy_amount = 3
for _ in range(backref_amount - 3):
# Check our existing locations to figure out if we still have
# longest prefixes.
locations = self.locations[
self.data[self.read_pos + copy_amount]
]
new_backref_locations: List[int] = [
absolute_pos
for absolute_pos in possible_backref_locations
if absolute_pos + copy_amount in locations
]
# If we have no longest prefixes, that means that any of the
# previous prefixes are good enough.
if not new_backref_locations:
break
# Mark that we're copying an extra byte from the backref.
self.__ring_write(
self.data[
(self.read_pos + copy_amount) : (
self.read_pos + copy_amount + 1
)
]
)
copy_amount += 1
possible_backref_locations = new_backref_locations
# Now that we have a list of candidates, arbitrarily pick the
# first one as our candidate and output it.
absolute_pos = possible_backref_locations[0]
backref_pos = start_write_size - absolute_pos
lo = (copy_amount - 3) & 0xF | ((backref_pos & 0xF) << 4)
hi = (backref_pos >> 4) & 0xFF
flags |= self.FLAG_BACKREF << flagpos
data[flagpos] = bytes([hi, lo])
self.read_pos += copy_amount
self.left -= copy_amount
yield bytes([flags]) + b"".join(data[: (flagpos + 1)])
class Lz77:
"""
A wrapper class encapsulating Lz77 encoding and decoding.
"""
# The point at which we consider it better to trade off smaller data
# sent over the wire for a more computationally expensive compression.
REAL_COMPRESSION_THRESHOLD = 10 * 1024
def __init__(self, backref: Optional[int] = None) -> None:
"""
Initialize the object.
"""
self.backref = backref
def decompress(self, data: bytes) -> bytes:
"""
Given a binary blob, return a new binary blob representing the decompressed
data.
Parameters:
data - Lz77-compressed binary data
Returns:
Raw binary data.
"""
lz = Lz77Decompress(data, backref=self.backref)
return b"".join(lz.decompress_bytes())
def compress(self, data: bytes) -> bytes:
"""
Given a binary blob, return a new binary blob representing the compressed data.
Parameters:
data - Raw binary data.
Returns:
L7zz-compressed binary data.
"""
lz = Lz77Compress(data, backref=self.backref)
return b"".join(lz.compress_bytes())

8
v8_server/utils/xml.py
View File

@ -5,13 +5,13 @@ from time import time
from typing import Dict, Tuple
import lxml
import lzss
from flask import Request
from kbinxml import KBinXML
from lxml import etree as ET # noqa: N812
from v8_server.utils.arc4 import EamuseARC4
from v8_server.utils.eamuse import get_timestamp
from v8_server.utils.lz77 import Lz77
EAMUSE_CONFIG = {"encrypted": False, "compressed": False}
@ -37,7 +37,8 @@ def eamuse_read_xml(request: Request) -> Tuple[str, str, str, str, str]:
compress = headers["x-compress"] if "x-compress" in headers else None
if compress == "lz77":
xml_dec = lzss.decompress(xml_dec)
lz77 = Lz77()
xml_dec = lz77.decompress(xml_dec)
EAMUSE_CONFIG["compress"] = True
else:
EAMUSE_CONFIG["compress"] = False
@ -98,7 +99,8 @@ def eamuse_prepare_xml(xml: str) -> Tuple[bytes, Dict[str, str]]:
if EAMUSE_CONFIG["compress"]:
headers["X-Compress"] = "lz77"
# Actually do some compression here if we have to
lz77 = Lz77()
xml_bin = lz77.compress(xml_bin)
headers["X-Eamuse-Info"] = x_eamuse_info

24
v8_server/view/index.py
View File

@ -138,6 +138,30 @@ def package() -> Tuple[bytes, Dict[str, str]]:
return eamuse_prepare_xml(response)
@app.route("/local/service", methods=["POST"])
def local() -> Tuple[bytes, Dict[str, str]]:
xml, model, module, method, command = eamuse_read_xml(request)
if module == "shopinfo":
if method == "regist":
response = E.response(
E.shopinfo(
E.data(
E.cabid("1", {"__type": "u32"}),
E.locationid("nowhere"),
E.is_send("1", {"__type": "u8"}),
)
)
)
elif module == "demodata":
if method == "get":
response = E.response()
else:
response = base_response(module)
return eamuse_prepare_xml(response)
@app.route("/service/services/services/", methods=["POST"])
def services() -> Tuple[bytes, Dict[str, str]]:
# We don't need to actually read the data here, but let's do it anyway as it saves a