From 4e43b90156b5ff31ff273ba8afb5aeef3859f6eb Mon Sep 17 00:00:00 2001 From: tritoch Date: Fri, 30 Jun 2017 09:42:36 -0500 Subject: [PATCH] Write AllSets.json Don't scrape scryfall if we disable comparison Toggle for Dumping Error log --- main.py | 14 +++++++++----- spoilers.py | 6 +++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index 90b124af..c17319b0 100644 --- a/main.py +++ b/main.py @@ -17,7 +17,8 @@ presets = { "imageorder": ['wotc','scryfall','mtgs','mythicspoiler'], # prioritize images from certain sources "useexclusively": '', # if we *only* want to use one site TODO "dumpXML": False, # let travis print XML for testing - "scryfallComparison": False #if we want to debug compare scryfall to other sources, enable + "scryfallComparison": False, #if we want to debug compare scryfall to other sources, enable + "dumpErrors": True # print the error log from out/errors.json } try: @@ -66,8 +67,9 @@ def parseargs(): print "Setting preset " + preset + " to value " + str(argvalue) def save_allsets(AllSets): - #TODO Create AllSets.json for Oracle - print "" + with io.open('out/AllSets.json', 'w', encoding='utf8') as json_file: + data = json.dumps(AllSets, ensure_ascii=False, encoding='utf8', indent=2, sort_keys=True, separators=(',',':')) + json_file.write(unicode(data)) def save_masterpieces(masterpieces, setinfo): with open('out/' + setinfo['masterpieces']['setname'] + '.json', 'w') as outfile: @@ -101,9 +103,10 @@ if __name__ == '__main__': mtgs = spoilers.scrape_mtgs('http://www.mtgsalvation.com/spoilers.rss') #scrape mtgs rss feed [mtgs, split_cards] = spoilers.parse_mtgs(mtgs, [], [], [], presets['split_cards']) #parse spoilers into mtgjson format mtgs = spoilers.correct_cards(mtgs, manual_sets[setinfo['setname']]['cards'], card_corrections, delete_cards) #fix using the fixfiles - scryfall = spoilers.get_scryfall('https://api.scryfall.com/cards/search?q=++e:' + setinfo['setname'].lower()) mtgjson = spoilers.get_image_urls(mtgs, presets['isfullspoil'], setinfo['setname'], setinfo['setlongname'], setinfo['setsize'], setinfo) #get images if presets['scryfallComparison']: + scryfall = spoilers.get_scryfall( + 'https://api.scryfall.com/cards/search?q=++e:' + setinfo['setname'].lower()) mtgjson = spoilers.smash_mtgs_scryfall(mtgs, scryfall) if 'fullSpoil' in setinfo and setinfo['fullSpoil']: wotc = spoilers.scrape_fullspoil('', setinfo) @@ -132,13 +135,14 @@ if __name__ == '__main__': save_xml(spoilers.pretty_xml('out/spoiler.xml'), 'out/spoiler.xml') errorlog = spoilers.remove_corrected_errors(errorlog, card_corrections) save_errorlog(errorlog) - #save_allsets(AllSets) + save_allsets(AllSets) #save_setjson(mtgjson) if presets['dumpXML']: print '' with open('out/spoiler.xml', 'r') as xmlfile: print xmlfile.read() print '' + if presets['dumpErrors']: if errorlog != {}: print '//----- DUMPING ERROR LOG -----' print json.dumps(errorlog, ensure_ascii=False, encoding='utf8', indent=2, sort_keys=True, separators=(',',':')) diff --git a/spoilers.py b/spoilers.py index 466b6c06..ea83fba9 100644 --- a/spoilers.py +++ b/spoilers.py @@ -521,7 +521,7 @@ def convert_scryfall(scryfall): cards2 = [] for card in scryfall: card2 = {} - card2['cmc'] = int((card['cmc']).split('.')[0]) + card2['cmc'] = int(card['cmc']) if card.has_key('mana_cost'): card2['manaCost'] = card['mana_cost'].replace('{','').replace('}','') else: @@ -1304,8 +1304,8 @@ def get_allsets(): version = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko / 20071127 Firefox / 2.0.0.11' opener = MyOpener() - opener.retrieve('http://mtgjson.com/json/AllSets.json', 'AllSets.pre.json') - with open('AllSets.pre.json') as data_file: + opener.retrieve('http://mtgjson.com/json/AllSets.json', 'AllSets.json') + with open('AllSets.json') as data_file: AllSets = json.load(data_file) return AllSets