From ca6f985024689e575a032c0248cfe0faea182509 Mon Sep 17 00:00:00 2001 From: Alex Xu Date: Sun, 2 Oct 2022 20:20:57 -0400 Subject: [PATCH 1/2] use multithreading to fetch json faster --- s3s.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/s3s.py b/s3s.py index 9be9ae9..af6d002 100644 --- a/s3s.py +++ b/s3s.py @@ -11,6 +11,8 @@ from PIL import Image from io import BytesIO import iksm, utils#, utils_ss +from concurrent.futures import ThreadPoolExecutor + A_VERSION = "0.1.5" DEBUG = False @@ -50,6 +52,8 @@ BULLETTOKEN = CONFIG_DATA["bullettoken"] # for accessing splatnet - base64 j SESSION_TOKEN = CONFIG_DATA["session_token"] # for nintendo login F_GEN_URL = CONFIG_DATA["f_gen"] # endpoint for generating f (imink API by default) +thread_pool = ThreadPoolExecutor(max_workers=10) + # SET HTTP HEADERS if "app_user_agent" in CONFIG_DATA: APP_USER_AGENT = str(CONFIG_DATA["app_user_agent"]) @@ -259,23 +263,10 @@ def fetch_json(which, separate=False, exportall=False, specific=False, numbers_o ink_list.extend(battle_ids) salmon_list.extend(job_ids) else: # ALL DATA - TAKES A LONG TIME - for bid in battle_ids: - query2_b = requests.post(utils.GRAPHQL_URL, - data=utils.gen_graphql_body(utils.translate_rid["VsHistoryDetailQuery"], "vsResultId", bid), - headers=headbutt(), - cookies=dict(_gtoken=GTOKEN)) - query2_resp_b = json.loads(query2_b.text) - ink_list.append(query2_resp_b) - swim() - for jid in job_ids: - query2_j = requests.post(utils.GRAPHQL_URL, - data=utils.gen_graphql_body(utils.translate_rid["CoopHistoryDetailQuery"], "coopHistoryDetailId", jid), - headers=headbutt(), - cookies=dict(_gtoken=GTOKEN)) - query2_resp_j = json.loads(query2_j.text) - salmon_list.append(query2_resp_j) - swim() + ink_list.extend(thread_pool.map(fetch_detailed_result, [True]*len(battle_ids), battle_ids, [swim]*len(battle_ids))) + + salmon_list.extend(thread_pool.map(fetch_detailed_result, [False]*len(job_ids), job_ids, [swim]*len(job_ids))) if needs_sorted: # put regular, bankara, and private in order, since they were exported in sequential chunks try: @@ -301,6 +292,16 @@ def fetch_json(which, separate=False, exportall=False, specific=False, numbers_o combined = ink_list + salmon_list return combined +def fetch_detailed_result(is_vs_history, history_id, swim): + sha = "VsHistoryDetailQuery" if is_vs_history else "CoopHistoryDetailQuery" + varname = "vsResultId" if is_vs_history else "coopHistoryDetailId" + query2 = requests.post(utils.GRAPHQL_URL, + data=utils.gen_graphql_body(utils.translate_rid[sha], varname, history_id), + headers=headbutt(), + cookies=dict(_gtoken=GTOKEN)) + query2_resp = json.loads(query2.text) + swim() + return query2_resp def update_salmon_profile(): ''' Updates stat.ink Salmon Run stats/profile.''' @@ -1220,6 +1221,8 @@ def main(): for hash in results: fetch_and_upload_single_result(hash, noun, False, blackout, test_run) # monitoring mode = False + thread_pool.shutdown(wait=True) + if __name__ == "__main__": main() From ed944ed3771ed8945db688c678f7c7d616704f39 Mon Sep 17 00:00:00 2001 From: Alex Xu Date: Thu, 17 Nov 2022 10:20:25 -0500 Subject: [PATCH 2/2] Limit number of threads to 2 --- s3s.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3s.py b/s3s.py index 90e7766..7bb1b73 100644 --- a/s3s.py +++ b/s3s.py @@ -49,7 +49,7 @@ BULLETTOKEN = CONFIG_DATA["bullettoken"] # for accessing splatnet - base64 SESSION_TOKEN = CONFIG_DATA["session_token"] # for nintendo login F_GEN_URL = CONFIG_DATA["f_gen"] # endpoint for generating f (imink API by default) -thread_pool = ThreadPoolExecutor(max_workers=10) +thread_pool = ThreadPoolExecutor(max_workers=2) # SET HTTP HEADERS DEFAULT_USER_AGENT = 'Mozilla/5.0 (Linux; Android 11; Pixel 5) ' \