diff --git a/s3s.py b/s3s.py index 966576a..e6f4df6 100755 --- a/s3s.py +++ b/s3s.py @@ -5,6 +5,7 @@ # License: GPLv3 import argparse, base64, datetime, json, os, shutil, re, requests, sys, time, uuid +from concurrent.futures import ThreadPoolExecutor from subprocess import call import msgpack from packaging import version @@ -49,6 +50,8 @@ BULLETTOKEN = CONFIG_DATA["bullettoken"] # for accessing splatnet - base64 SESSION_TOKEN = CONFIG_DATA["session_token"] # for nintendo login F_GEN_URL = CONFIG_DATA["f_gen"] # endpoint for generating f (imink API by default) +thread_pool = ThreadPoolExecutor(max_workers=2) + # SET HTTP HEADERS DEFAULT_USER_AGENT = 'Mozilla/5.0 (Linux; Android 11; Pixel 5) ' \ 'AppleWebKit/537.36 (KHTML, like Gecko) ' \ @@ -279,23 +282,9 @@ def fetch_json(which, separate=False, exportall=False, specific=False, numbers_o ink_list.extend(battle_ids) salmon_list.extend(job_ids) else: # ALL DATA - TAKES A LONG TIME - for bid in battle_ids: - query2_b = requests.post(utils.GRAPHQL_URL, - data=utils.gen_graphql_body(utils.translate_rid["VsHistoryDetailQuery"], "vsResultId", bid), - headers=headbutt(), - cookies=dict(_gtoken=GTOKEN)) - query2_resp_b = json.loads(query2_b.text) - ink_list.append(query2_resp_b) - swim() + ink_list.extend(thread_pool.map(fetch_detailed_result, [True]*len(battle_ids), battle_ids, [swim]*len(battle_ids))) - for jid in job_ids: - query2_j = requests.post(utils.GRAPHQL_URL, - data=utils.gen_graphql_body(utils.translate_rid["CoopHistoryDetailQuery"], "coopHistoryDetailId", jid), - headers=headbutt(forcelang='en-US'), - cookies=dict(_gtoken=GTOKEN)) - query2_resp_j = json.loads(query2_j.text) - salmon_list.append(query2_resp_j) - swim() + salmon_list.extend(thread_pool.map(fetch_detailed_result, [False]*len(job_ids), job_ids, [swim]*len(job_ids))) if needs_sorted: # put regular, bankara, and private in order, since they were exported in sequential chunks try: @@ -321,6 +310,22 @@ def fetch_json(which, separate=False, exportall=False, specific=False, numbers_o combined = ink_list + salmon_list return combined +def fetch_detailed_result(is_vs_history, history_id, swim): + '''Helper function for fetch_json().''' + + sha = "VsHistoryDetailQuery" if is_vs_history else "CoopHistoryDetailQuery" + varname = "vsResultId" if is_vs_history else "coopHistoryDetailId" + lang = None if is_vs_history else 'en-US' + + query2 = requests.post(utils.GRAPHQL_URL, + data=utils.gen_graphql_body(utils.translate_rid[sha], varname, history_id), + headers=headbutt(forcelang=lang), + cookies=dict(_gtoken=GTOKEN)) + query2_resp = json.loads(query2.text) + + swim() + return query2_resp + def update_salmon_profile(): ''' Updates stat.ink Salmon Run stats/profile.''' @@ -1754,6 +1759,8 @@ def main(): for hash in results: fetch_and_upload_single_result(hash, noun, blackout, test_run) # not monitoring mode + thread_pool.shutdown(wait=True) + if __name__ == "__main__": main()