Leahnaya/s3s
1
0
Fork 0
mirror of https://github.com/frozenpandaman/s3s.git synced 2026-09-09 02:55:12 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

use multithreading to fetch json faster (#41)

Browse Source 
Co-authored-by: eli <frozenpandaman@users.noreply.github.com>
This commit is contained in:
Akira
2022-11-25 12:02:53 -05:00
committed by GitHub
parent 1754f98c70
commit 08b887277a
1 changed files with 24 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
s3s.py
View File

@@ -5,6 +5,7 @@
# License: GPLv3
import argparse, base64, datetime, json, os, shutil, re, requests, sys, time, uuid
from concurrent.futures import ThreadPoolExecutor
from subprocess import call
import msgpack
from packaging import version
@@ -49,6 +50,8 @@ BULLETTOKEN = CONFIG_DATA["bullettoken"] # for accessing splatnet - base64
SESSION_TOKEN = CONFIG_DATA["session_token"] # for nintendo login
F_GEN_URL = CONFIG_DATA["f_gen"] # endpoint for generating f (imink API by default)
thread_pool = ThreadPoolExecutor(max_workers=2)
# SET HTTP HEADERS
DEFAULT_USER_AGENT = 'Mozilla/5.0 (Linux; Android 11; Pixel 5) ' \
'AppleWebKit/537.36 (KHTML, like Gecko) ' \
@@ -279,23 +282,9 @@ def fetch_json(which, separate=False, exportall=False, specific=False, numbers_o
ink_list.extend(battle_ids)
salmon_list.extend(job_ids)
else: # ALL DATA - TAKES A LONG TIME
for bid in battle_ids:
query2_b = requests.post(utils.GRAPHQL_URL,
data=utils.gen_graphql_body(utils.translate_rid["VsHistoryDetailQuery"], "vsResultId", bid),
headers=headbutt(),
cookies=dict(_gtoken=GTOKEN))
query2_resp_b = json.loads(query2_b.text)
ink_list.append(query2_resp_b)
swim()
ink_list.extend(thread_pool.map(fetch_detailed_result, [True]*len(battle_ids), battle_ids, [swim]*len(battle_ids)))
for jid in job_ids:
query2_j = requests.post(utils.GRAPHQL_URL,
data=utils.gen_graphql_body(utils.translate_rid["CoopHistoryDetailQuery"], "coopHistoryDetailId", jid),
headers=headbutt(forcelang='en-US'),
cookies=dict(_gtoken=GTOKEN))
query2_resp_j = json.loads(query2_j.text)
salmon_list.append(query2_resp_j)
swim()
salmon_list.extend(thread_pool.map(fetch_detailed_result, [False]*len(job_ids), job_ids, [swim]*len(job_ids)))
if needs_sorted: # put regular, bankara, and private in order, since they were exported in sequential chunks
try:
@@ -321,6 +310,22 @@ def fetch_json(which, separate=False, exportall=False, specific=False, numbers_o
combined = ink_list + salmon_list
return combined
def fetch_detailed_result(is_vs_history, history_id, swim):
'''Helper function for fetch_json().'''
sha = "VsHistoryDetailQuery" if is_vs_history else "CoopHistoryDetailQuery"
varname = "vsResultId" if is_vs_history else "coopHistoryDetailId"
lang = None if is_vs_history else 'en-US'
query2 = requests.post(utils.GRAPHQL_URL,
data=utils.gen_graphql_body(utils.translate_rid[sha], varname, history_id),
headers=headbutt(forcelang=lang),
cookies=dict(_gtoken=GTOKEN))
query2_resp = json.loads(query2.text)
swim()
return query2_resp
def update_salmon_profile():
''' Updates stat.ink Salmon Run stats/profile.'''
@@ -642,7 +647,7 @@ def prepare_battle_result(battle, ismonitoring, isblackout, overview_data=None):
full_rank = re.split('([0-9]+)', child["udemae"].lower())
was_s_plus_before = len(full_rank) > 1 # true if "before" rank is s+
payload["rank_before"] = full_rank[0]
if was_s_plus_before:
payload["rank_before_s_plus"] = int(full_rank[1])
@@ -1754,6 +1759,8 @@ def main():
for hash in results:
fetch_and_upload_single_result(hash, noun, blackout, test_run) # not monitoring mode
thread_pool.shutdown(wait=True)
if __name__ == "__main__":
main()