From 4ee8c132eb39a58eb26cb98931f655a476ed53dc Mon Sep 17 00:00:00 2001 From: tritoch Date: Mon, 19 Jun 2017 14:31:38 -0500 Subject: [PATCH 1/4] Improved Split Card Handling. Dump XML to log. --- main.py | 44 +++++++++++++++++++------------------------- spoilers.py | 28 +++++++++++----------------- 2 files changed, 30 insertions(+), 42 deletions(-) diff --git a/main.py b/main.py index ed4791b9..7016344c 100644 --- a/main.py +++ b/main.py @@ -1,19 +1,19 @@ # -*- coding: utf-8 -*- import spoilers -import sys import os -import shutil -#import configparser +import commentjson import json -#import urllib presets = { - "isfullspoil": False, #when full spoil comes around, we only want to use WOTC images - "includeMasterpieces": True, #if the set has masterpieces, let's get those too - "oldRSS": False, #maybe MTGS hasn't updated their spoiler.rss but new cards have leaked + "isfullspoil": False, # when full spoil comes around, we only want to use WOTC images + "includeMasterpieces": True, # if the set has masterpieces, let's get those too + "oldRSS": False, # maybe MTGS hasn't updated their spoiler.rss but new cards have leaked "split_cards": { "Grind": "Dust" - } + }, + "siteorder": ['scryfall','mtgs','mythicspoiler'], # if we want to use one site before another for card data TODO + "useexclusively": '', # if we *only* want to use one site TODO + "dumpXML": True # let travis print XML for teting } with open('set_info.json') as data_file: @@ -21,7 +21,6 @@ with open('set_info.json') as data_file: with open('cards_manual.json') as data_file: manual_sets = json.load(data_file) - #manual_cards = manual_cards['cards'] with open('cards_corrections.json') as data_file: card_corrections = json.load(data_file) @@ -33,22 +32,11 @@ errorlog = [] #TODO insert configparser to add config.ini file -# comment this out for now because multiple sets -#for argument in sys.argv: - #we can modify any of the variables from the set infos file or the presets above at runtime - #works only for first-level variables currently (editing masterpieces - #syntax is variable="new value" - #for setinfo in setinfos: - #for setinfo in setinfo: - # if setinfo in argument.split("=")[0]: - # setinfos[setinfo] = argument.split("=")[1] - #for preset in presets: - # if preset in argument.split("=")[0]: - # presets[preset] = argument.split("=")[1] +#TODO parse arguments so we can have Travis print xml for testing def save_allsets(AllSets): #TODO Create AllSets.json for Oracle - print "Saving AllSets" + print "" def save_masterpieces(masterpieces, setinfo): with open('out/' + setinfo['masterpieces']['setname'] + '.json', 'w') as outfile: @@ -66,7 +54,7 @@ def save_errorlog(errorlog): fixederrors.append(error) else: unfixederrors.append(error) - errorlog = {"unfixed": unfixederrors, "fixed": fixederrors} + errorlog = [unfixederrors] with open('out/errors.json', 'w') as outfile: json.dump(errorlog, outfile, sort_keys=True, indent=2, separators=(',', ': ')) @@ -86,7 +74,7 @@ if __name__ == '__main__': mtgs = { "cards":[] } else: mtgs = spoilers.scrape_mtgs('http://www.mtgsalvation.com/spoilers.rss') #scrape mtgs rss feed - mtgs = spoilers.parse_mtgs(mtgs, [], [], [], presets['split_cards']) #parse spoilers into mtgjson format + [mtgs, split_cards] = spoilers.parse_mtgs(mtgs, [], [], [], presets['split_cards']) #parse spoilers into mtgjson format mtgs = spoilers.correct_cards(mtgs, manual_sets[setinfo['setname']]['cards'], card_corrections, delete_cards) #fix using the fixfiles scryfall = spoilers.get_scryfall('https://api.scryfall.com/cards/search?q=++e:' + setinfo['setname'].lower()) mtgs = spoilers.get_image_urls(mtgs, presets['isfullspoil'], setinfo['setname'], setinfo['setlongname'], setinfo['setsize']) #get images @@ -114,5 +102,11 @@ if __name__ == '__main__': spoilers.write_combined_xml(combinedjson, setinfos) save_xml(spoilers.pretty_xml('out/spoiler.xml'), 'out/spoiler.xml') save_errorlog(errorlog) - save_allsets(AllSets) + #save_allsets(AllSets) #save_setjson(mtgjson) + if presets['dumpXML']: + print '----- DUMPING SPOILER.XML -----' + with open('out/spoiler.xml', 'r') as xmlfile: + print xmlfile.read() + print '----- END XML DUMP -----' + diff --git a/spoilers.py b/spoilers.py index 8b98913f..3fadc2ac 100644 --- a/spoilers.py +++ b/spoilers.py @@ -72,6 +72,8 @@ def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], spli card2['setnumber'] = card['setnumber'] + 'b' if 'rarity' in card: card2['rarity'] = card['rarity'] + if not card1['name'] in split_cards: + split_cards[card1['name']] = card2['name'] cards2.append(card1) cards2.append(card2) else: @@ -241,7 +243,7 @@ def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], spli cardarray.append(cardjson) - return {"cards": cardarray} + return [{"cards": cardarray}, split_cards] def correct_cards(mtgjson, manual_cards=[], card_corrections=[], delete_cards=[]): mtgjson2 = [] @@ -816,26 +818,18 @@ def write_xml(mtgjson, setname, setlongname, setreleasedate, split_cards=[]): cardtype = card["type"] if card.has_key("names"): if "layout" in card: - if card["layout"] != 'split': - if len(card["names"]) > 1: - if card["names"][0] == card["name"]: - related = card["names"][1] - text += '\n\n(Related: ' + card["names"][1] + ')' - dfccount += 1 - elif card['names'][1] == card['name']: - related = card["names"][0] - text += '\n\n(Related: ' + card["names"][0] + ')' - else: - for carda in split_cards: - if card["name"] == carda: - cardb = split_cards[carda] + if card['layout'] == 'split': + if 'names' in card: + if card['name'] == card['names'][0]: for jsoncard in mtgjson["cards"]: - if cardb == jsoncard["name"]: + if jsoncard['name'] == card['names'][1]: cardtype += " // " + jsoncard["type"] manacost += " // " + (jsoncard["manaCost"]).replace('{', '').replace('}', '') cardcmc += " // " + str(jsoncard["cmc"]) text += "\n---\n" + jsoncard["text"] - name += " // " + cardb + name += " // " + jsoncard['name'] + else: + print card["name"] + " has names, but layout != split" else: print card["name"] + " has multiple names and no 'layout' key" @@ -890,7 +884,7 @@ def write_xml(mtgjson, setname, setlongname, setreleasedate, split_cards=[]): cardsxml.write("\n") - print 'XML STATS' + print 'XML Stats for ' + setlongname print 'Total cards: ' + str(count) if dfccount > 0: print 'DFC: ' + str(dfccount) From ef4f2c73771c082d16bafde2b86c0d3407b423de Mon Sep 17 00:00:00 2001 From: tritoch Date: Mon, 19 Jun 2017 14:49:00 -0500 Subject: [PATCH 2/4] Improved Split Card Handling. Dump XML to log. --- main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.py b/main.py index 7016344c..c17de014 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import spoilers import os -import commentjson import json presets = { @@ -13,7 +12,7 @@ presets = { }, "siteorder": ['scryfall','mtgs','mythicspoiler'], # if we want to use one site before another for card data TODO "useexclusively": '', # if we *only* want to use one site TODO - "dumpXML": True # let travis print XML for teting + "dumpXML": True # let travis print XML for testing } with open('set_info.json') as data_file: From 4f4955975185f005ff5112e616ec44ed5ad9f666 Mon Sep 17 00:00:00 2001 From: tritoch Date: Mon, 19 Jun 2017 15:57:11 -0500 Subject: [PATCH 3/4] Missed file extension --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index b081e77b..8552ba8f 100644 --- a/main.py +++ b/main.py @@ -19,7 +19,7 @@ presets = { with open('set_info') as data_file: setinfos = commentjson.load(data_file) -with open('cards_manual.json') as data_file: +with open('cards_manual') as data_file: manual_sets = json.load(data_file) with open('cards_corrections') as data_file: From f7e71f4ca1e883b419096151c8bc668debe70d8c Mon Sep 17 00:00:00 2001 From: tooomm Date: Mon, 19 Jun 2017 23:23:38 +0200 Subject: [PATCH 4/4] full list of supported services + rephrasing (#11) [skip ci] * full list of supported services + rephrasing * Update README.md * [skip ci] final wording --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f5a9dec4..58eea140 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Magic-Spoiler [![Gitter Chat](https://img.shields.io/gitter/room/Cockatrice/Magic-Spoiler.svg)](https://gitter.im/Cockatrice/Magic-Spoiler) # -Magic-Spoiler is a python script to scrape MTGS, Scryfall, and Wizards.com to compile a cockatrice-friendly XML file as well as json files. +Magic-Spoiler is a python script to scrape MTG Salvation, Scryfall, MythicSpoiler and Wizards to compile a XML file (Cockatrice formatted) and a general application ready json file (mtgjson formatted). ## Output [![Build Status](https://travis-ci.org/Cockatrice/Magic-Spoiler.svg?branch=master)](https://travis-ci.org/Cockatrice/Magic-Spoiler) ## Just looking for XML or JSON files? [They're in our files branch!](https://github.com/Cockatrice/Magic-Spoiler/tree/files)