Leahnaya/Magic-Spoiler
1
0
Fork 0
mirror of https://github.com/Cockatrice/Magic-Spoiler.git synced 2026-03-21 17:54:59 -05:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Python3 conversion and Update for MTGJSONv4

Browse Source
This commit is contained in:
Zach Halpern 2018-12-21 16:08:26 -05:00
parent aad0479ca1
commit 0b021b377f
6 changed files with 136 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
main.py
View File

@ -17,10 +17,10 @@ presets = {
"isfullspoil": False, # when full spoil comes around, we only want to use WOTC images
"includeMasterpieces": True, # if the set has masterpieces, let's get those too
"oldRSS": False, # maybe MTGS hasn't updated their spoiler.rss but new cards have leaked
"dumpXML": False, # let travis print XML for testing
"dumpXML": False, # let travis print (XML for testing)
# only use Scryfall data (no mtgs for ANY sets)
"scryfallOnly": False,
"dumpErrors": True # print the error log from out/errors.json
"dumpErrors": True # print (the error log from out/errors.json)
}
setinfos = verify_files.load_file('set_info.yml','yaml_multi')
@ -43,27 +43,24 @@ def parseargs():
elif argvalue in ['false', 'False', 'F', 'f']:
argvalue = False
presets[preset] = argvalue
print "Setting preset " + preset + " to value " + str(argvalue)
print("Setting preset " + preset + " to value " + str(argvalue))
def save_allsets(AllSets):
with io.open('out/AllSets.json', 'w', encoding='utf8') as json_file:
data = json.dumps(AllSets, ensure_ascii=False, encoding='utf8',
indent=2, sort_keys=True, separators=(',', ':'))
json_file.write(unicode(data))
data = json.dumps(AllSets, ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ':'))
json_file.write(data)
def save_masterpieces(masterpieces, setinfo):
with open('out/' + setinfo['masterpieces']['code'] + '.json', 'w') as outfile:
json.dump(masterpieces, outfile, sort_keys=True,
indent=2, separators=(',', ': '))
json.dump(masterpieces, outfile, sort_keys=True, indent=2, separators=(',', ': '))
def save_setjson(mtgs, filename):
with io.open('out/' + filename + '.json', 'w', encoding='utf8') as json_file:
data = json.dumps(mtgs, ensure_ascii=False, encoding='utf8',
indent=2, sort_keys=True, separators=(',', ':'))
json_file.write(unicode(data))
data = json.dumps(mtgs, ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ':'))
json_file.write(data)
def save_errorlog(errorlog):
@ -77,31 +74,31 @@ def save_xml(xmlstring, outfile):
else:
append_or_write = 'w'
with open(outfile, append_or_write) as xmlfile:
xmlfile.write(xmlstring.encode('utf-8'))
xmlfile.write(xmlstring)
def verify_xml(file, schema):
try:
schema_doc = etree.fromstring(schema)
except Exception as e:
print "XSD for " + file + " is invalid"
print schema
print e
print ("XSD for " + file + " is invalid")
print (schema)
print (e)
return False
xml_schema = etree.XMLSchema(schema_doc)
try:
xml_doc = etree.parse(file)
except Exception as e:
print "XML file " + file + " is invalid"
print e
print ("XML file " + file + " is invalid")
print (e)
return False
try:
xml_schema.assert_(xml_doc)
except:
xsd_errors = xml_schema.error_log
print "Errors validating XML file " + file + " against XSD:"
print ("Errors validating XML file " + file + " against XSD:")
for error in xsd_errors:
print error
print (error)
sys.exit("Error: " + file + " does not pass Cockatrice XSD validation.")
return False
return True
@ -112,10 +109,13 @@ if __name__ == '__main__':
AllSets = spoilers.get_allsets() # get AllSets from mtgjson
combinedjson = {}
noCards = []
del AllSets['RNA']
for setinfo in setinfos:
if setinfo['code'] in AllSets:
print "Found " +setinfo['code']+ " set from set_info.yml in MTGJSON, not adding it"
print ("Found " +setinfo['code']+ " set from set_info.yml in MTGJSON, not adding it")
continue
print("Handling {}".format(setinfo['code']))
if presets['oldRSS'] or 'noRSS' in setinfo and setinfo['noRSS']:
mtgs = {"cards": []}
else:
@ -174,22 +174,22 @@ if __name__ == '__main__':
save_xml(spoilers.pretty_xml('out/spoiler.xml'), 'out/spoiler.xml')
cockatrice_xsd = requests.get('https://raw.githubusercontent.com/Cockatrice/Cockatrice/master/doc/cards.xsd').text
if verify_xml('out/spoiler.xml', cockatrice_xsd): # check if our XML passes Cockatrice's XSD
print 'spoiler.xml passes Cockatrice XSD verification'
print ('spoiler.xml passes Cockatrice XSD verification')
else:
print 'spoiler.xml fails Cockatrice XSD verification'
print ('spoiler.xml fails Cockatrice XSD verification')
errorlog = spoilers.remove_corrected_errors(errorlog, card_corrections)
save_errorlog(errorlog)
save_allsets(AllSets)
# save_setjson(mtgjson)
if presets['dumpXML']:
print '<!----- DUMPING SPOILER.XML -----!>'
print ('<!----- DUMPING SPOILER.XML -----!>')
with open('out/spoiler.xml', 'r') as xmlfile:
print xmlfile.read()
print '<!----- END XML DUMP -----!>'
print (xmlfile.read())
print ('<!----- END XML DUMP -----!>')
if presets['dumpErrors']:
if errorlog != {}:
print '//----- DUMPING ERROR LOG -----'
print yaml.safe_dump(errorlog, default_flow_style=False)
print '//----- END ERROR LOG -----'
print ('//----- DUMPING ERROR LOG -----')
print (yaml.safe_dump(errorlog, default_flow_style=False))
print ('//----- END ERROR LOG -----')
else:
print "No Detected Errors!"
print ("No Detected Errors!")

8
mtgs_scraper.py
View File

@ -38,7 +38,7 @@ def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], rela
gallery_list = list_mtgs_gallery(setinfo['mtgsurl'])
for card in cards:
if card['name'] not in gallery_list:
print "Removing card scraped from MTGS RSS but not in their gallery: " + card['name']
print ("Removing card scraped from MTGS RSS but not in their gallery: " + card['name'])
cards.remove(card)
for card in cards:
@ -54,7 +54,7 @@ def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], rela
htmltags = re.compile(r'<.*?>')
card['rules'] = htmltags.sub('', card['rules'])
if '//' in card['name'] or 'Aftermath' in card['rules']:
print 'Splitting up Aftermath card ' + card['name']
print ('Splitting up Aftermath card ' + card['name'])
card1 = card.copy()
card2 = dict(cost='', cmc='', img='', pow='', name='', rules='', type='',
color='', altname='', colorIdentity='', colorArray=[], colorIdentityArray=[], setnumber='', rarity='')
@ -177,7 +177,7 @@ def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], rela
if 'number' in card:
if 'b' in card['number'] or 'a' in card['number']:
if not 'layout' in card:
print card['name'] + " has a a/b number but no 'layout'"
print (card['name'] + " has a a/b number but no 'layout'")
card['type'] = card['type'].replace('instant', 'Instant').replace(
'sorcery', 'Sorcery').replace('creature', 'Creature')
if '-' in card['type']:
@ -201,7 +201,7 @@ def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], rela
# may force 'special' in the future
if card['rarity'] not in ['Mythic Rare', 'Rare', 'Uncommon', 'Common', 'Special', 'Basic Land']:
#errors.append({"name": card['name'], "key": "rarity", "value": card['rarity']})
print card['name'] + ' has rarity = ' + card['rarity']
print (card['name'] + ' has rarity = ' + card['rarity'])
if subtypes:
cardjson['subtypes'] = subtypes
cardjson["rarity"] = card['rarity']

73
scryfall_scraper.py
View File

@ -12,20 +12,20 @@ def get_scryfall(setUrl='https://api.scryfall.com/cards/search?q=++e:xln'):
while setDone == False:
setcards = requests.get(setUrl)
setcards = setcards.json()
if setcards.has_key('data'):
if 'data' in setcards.keys():
scryfall.append(setcards['data'])
else:
setDone = True
print 'No Scryfall data'
print ('No Scryfall data')
scryfall = ['']
time.sleep(.1) # 100ms sleep, see "Rate Limits and Good Citizenship" at https://scryfall.com/docs/api
if setcards.has_key('has_more'):
if 'has_more' in setcards.keys():
if setcards['has_more']:
setUrl = setcards['next_page']
else:
setDone = True
else:
print 'Scryfall does not "has_more"'
print ('Scryfall does not "has_more"')
setDone = True
if not scryfall[0] == '':
import json
@ -38,6 +38,10 @@ def get_scryfall(setUrl='https://api.scryfall.com/cards/search?q=++e:xln'):
else:
return {'cards': []}
def merge_two_dicts(x, y):
z = x.copy() # start with x's keys and values
z.update(y) # modifies z with y's keys and values & returns None
return z
def convert_scryfall(scryfall):
cards2 = []
@ -53,8 +57,8 @@ def convert_scryfall(scryfall):
cardNoFaces[key] = card[key]
cardNoFaces['layout'] = 'double-faced'
cardNoFaces['names'] = [card['card_faces'][0]['name'], card['card_faces'][1]['name']]
card1 = dict(cardNoFaces.items() + card['card_faces'][0].items())
card2 = dict(cardNoFaces.items() + card['card_faces'][1].items())
card1 = merge_two_dicts(cardNoFaces, card['card_faces'][0])
card2 = merge_two_dicts(cardNoFaces, card['card_faces'][1])
card1['collector_number'] = card1['collector_number'] + 'a'
card2['collector_number'] = card2['collector_number'] + 'b'
scryfall2.append(card1)
@ -64,9 +68,14 @@ def convert_scryfall(scryfall):
for key in card:
if key != 'card_faces':
cardNoFaces[key] = card[key]
cardNoFaces['names'] = [card['card_faces'][0]['name'], card['card_faces'][1]['name']]
card1 = dict(cardNoFaces.items() + card['card_faces'][0].items())
card2 = dict(cardNoFaces.items() + card['card_faces'][1].items())
# print("CZZ: {}".format(cardNoFaces))
# print("CZZ: {}".format(card))
card1 = merge_two_dicts(cardNoFaces, card['card_faces'][0])
card2 = merge_two_dicts(cardNoFaces, card['card_faces'][1])
card1['collector_number'] = str(card['collector_number']) + "a"
card2['collector_number'] = str(card['collector_number']) + "b"
scryfall2.append(card1)
@ -81,7 +90,7 @@ def convert_scryfall(scryfall):
card2['cmc'] = int(card['cmc'])
if 'names' in card:
card2['names'] = card['names']
if card.has_key('mana_cost'):
if 'mana_cost' in card.keys():
card2['manaCost'] = card['mana_cost'].replace(
'{', '').replace('}', '')
else:
@ -90,7 +99,7 @@ def convert_scryfall(scryfall):
card2['number'] = card['collector_number']
card2['rarity'] = card['rarity'].replace(
'mythic', 'mythic rare').title()
if card.has_key('oracle_text'):
if 'oracle_text' in card.keys():
card2['text'] = card['oracle_text'].replace(
u"\u2014", '-').replace(u"\u2212", "-")
else:
@ -120,39 +129,39 @@ def convert_scryfall(scryfall):
if 'Basic Land' in card['type_line']:
card2['rarity'] = "Basic Land"
if 'Legendary' in card['type_line']:
if card2.has_key('supertypes'):
if 'supertypes' in card2.keys():
card2['supertypes'].append('Legendary')
else:
card2['supertypes'] = ['Legendary']
if 'Snow' in card['type_line']:
if card2.has_key('supertypes'):
if 'supertypes' in card2.keys():
card2['supertypes'].append('Snow')
else:
card2['supertypes'] = ['Snow']
if 'Elite' in card['type_line']:
if card2.has_key('supertypes'):
if 'supertypes' in card2.keys():
card2['supertypes'].append('Elite')
else:
card2['supertypes'] = ['Elite']
if 'Basic' in card['type_line']:
if card2.has_key('supertypes'):
if 'supertypes' in card2.keys():
card2['supertypes'].append('Basic')
else:
card2['supertypes'] = ['Basic']
if 'World' in card['type_line']:
if card2.has_key('supertypes'):
if 'supertypes' in card2.keys():
card2['supertypes'].append('World')
else:
card2['supertypes'] = ['World']
if 'Ongoing' in card['type_line']:
if card2.has_key('supertypes'):
if 'supertypes' in card2.keys():
card2['supertypes'].append('Ongoing')
else:
card2['supertypes'] = ['Ongoing']
card2['types'] = cardtypes
if card.has_key('color_identity'):
if 'color_identity' in card.keys():
card2['colorIdentity'] = card['color_identity']
if card.has_key('colors'):
if 'colors' in card.keys():
if not card['colors'] == []:
card2['colors'] = []
if 'W' in card['colors']:
@ -166,28 +175,28 @@ def convert_scryfall(scryfall):
if 'G' in card['colors']:
card2['colors'].append("Green")
#card2['colors'] = card['colors']
if card.has_key('all_parts'):
if'all_parts' in card.keys():
card2['names'] = []
for partname in card['all_parts']:
card2['names'].append(partname['name'])
if card.has_key('power'):
if'power' in card.keys():
card2['power'] = card['power']
if card.has_key('toughness'):
if'toughness' in card.keys():
card2['toughness'] = card['toughness']
if card.has_key('layout'):
if'layout' in card.keys():
if card['layout'] != 'normal':
card2['layout'] = card['layout']
if card.has_key('loyalty'):
if'loyalty' in card.keys():
card2['loyalty'] = card['loyalty']
if card.has_key('artist'):
if'artist' in card.keys():
card2['artist'] = card['artist']
# if card.has_key('source'):
# if'source' in card.keys():
# card2['source'] = card['source']
# if card.has_key('rulings'):
# if'rulings' in card.keys():
# card2['rulings'] = card['rulings']
if card.has_key('flavor_text'):
if'flavor_text' in card.keys():
card2['flavor'] = card['flavor_text']
if card.has_key('multiverse_id'):
if'multiverse_id' in card.keys():
card2['multiverseid'] = card['multiverse_id']
cards2.append(card2)
@ -204,19 +213,19 @@ def smash_mtgs_scryfall(mtgs, scryfall):
if key in mtgscard:
if not mtgscard[key] == scryfallcard[key]:
try:
print "%s's key %s\nMTGS : %s\nScryfall: %s" % (mtgscard['name'], key, mtgscard[key], scryfallcard[key])
print ("%s's key %s\nMTGS : %s\nScryfall: %s" % (mtgscard['name'], key, mtgscard[key], scryfallcard[key]))
except:
print "Error printing Scryfall vs MTGS debug info for " + mtgscard['name']
print ("Error printing Scryfall vs MTGS debug info for " + mtgscard['name'])
pass
cardFound = True
if not cardFound:
print "MTGS has card %s and Scryfall does not." % mtgscard['name']
print ("MTGS has card %s and Scryfall does not." % mtgscard['name'])
for scryfallcard in scryfall['cards']:
cardFound = False
for mtgscard in mtgs['cards']:
if scryfallcard['name'] == mtgscard['name']:
cardFound = True
if not cardFound:
print "Scryfall has card %s and MTGS does not." % scryfallcard['name']
print ("Scryfall has card %s and MTGS does not." % scryfallcard['name'])
return mtgs

95
spoilers.py
View File

@ -3,6 +3,7 @@ import requests
import re
import os
from lxml import html
import lzma
import datetime
import json
import mtgs_scraper
@ -89,7 +90,7 @@ def correct_cards(mtgjson, manual_cards=[], card_corrections=[], delete_cards=[]
mtgjson2.append(manualCard)
manual_added.append(manualCard['name'])
if manual_added != []:
print "Manual Cards Added: " + str(manual_added).strip('[]')
print ("Manual Cards Added: " + str(manual_added).strip('[]'))
mtgjson = {"cards": mtgjson2}
transforms = {}
@ -424,7 +425,7 @@ def write_xml(mtgjson, code, name, releaseDate):
"</set>\n"
"</sets>\n"
"<cards>\n")
# print mtgjson
# print (mtgjson)
for card in mtgjson["cards"]:
if 'names' in card:
if 'layout' in card and card['layout'] != 'double-faced':
@ -434,24 +435,24 @@ def write_xml(mtgjson, code, name, releaseDate):
newest = card["name"]
count += 1
name = card["name"]
if card.has_key("manaCost"):
if "manaCost" in card.keys():
manacost = card["manaCost"].replace('{', '').replace('}', '')
else:
manacost = ""
if card.has_key("power") or card.has_key("toughness"):
if "power" in card.keys() or "toughness" in card.keys():
if card["power"]:
pt = str(card["power"]) + "/" + str(card["toughness"])
else:
pt = 0
else:
pt = 0
if card.has_key("text"):
if "text" in card.keys():
text = card["text"]
else:
text = ""
cardcmc = str(card['cmc'])
cardtype = card["type"]
if card.has_key("names"):
if "names" in card.keys():
if "layout" in card:
if card['layout'] == 'split' or card['layout'] == 'aftermath':
if 'names' in card:
@ -470,15 +471,15 @@ def write_xml(mtgjson, code, name, releaseDate):
name += " // " + jsoncard['name']
elif card['layout'] == 'double-faced':
if not 'names' in card:
print card['name'] + ' is double-faced but no "names" key'
print (card['name'] + ' is double-faced but no "names" key')
else:
for dfcname in card['names']:
if dfcname != card['name']:
related = dfcname
else:
print card["name"] + " has names, but layout != split, aftermath, or double-faced"
print (card["name"] + " has names, but layout != split, aftermath, or double-faced")
else:
print card["name"] + " has multiple names and no 'layout' key"
print (card["name"] + " has multiple names and no 'layout' key")
tablerow = "1"
if "Land" in cardtype:
@ -494,17 +495,17 @@ def write_xml(mtgjson, code, name, releaseDate):
if 'b' in str(card['number']):
if 'layout' in card:
if card['layout'] == 'split' or card['layout'] == 'aftermath':
# print "We're skipping " + card['name'] + " because it's the right side of a split card"
# print ("We're skipping " + card['name'] + " because it's the right side of a split card")
continue
cardsxml.write("<card>\n")
cardsxml.write("<name>" + name.encode('utf-8') + "</name>\n")
cardsxml.write("<name>" + name + "</name>\n")
cardsxml.write(
'<set rarity="' + card['rarity'] + '" picURL="' + card["url"] + '">' + code + '</set>\n')
cardsxml.write(
"<manacost>" + manacost.encode('utf-8') + "</manacost>\n")
"<manacost>" + manacost + "</manacost>\n")
cardsxml.write("<cmc>" + cardcmc + "</cmc>\n")
if card.has_key('colors'):
if 'colors' in card.keys():
colorTranslate = {
"White": "W",
"Blue": "U",
@ -517,17 +518,17 @@ def write_xml(mtgjson, code, name, releaseDate):
'<color>' + colorTranslate[color] + '</color>\n')
if name + ' enters the battlefield tapped' in text:
cardsxml.write("<cipt>1</cipt>\n")
cardsxml.write("<type>" + cardtype.encode('utf-8') + "</type>\n")
cardsxml.write("<type>" + cardtype + "</type>\n")
if pt:
cardsxml.write("<pt>" + pt + "</pt>\n")
if card.has_key('loyalty'):
if 'loyalty' in card.keys():
cardsxml.write("<loyalty>" + str(card['loyalty']) + "</loyalty>\n")
cardsxml.write("<tablerow>" + tablerow + "</tablerow>\n")
cardsxml.write("<text>" + text.encode('utf-8') + "</text>\n")
cardsxml.write("<text>" + text + "</text>\n")
if related:
# for relatedname in related:
cardsxml.write(
"<related>" + related.encode('utf-8') + "</related>\n")
"<related>" + related + "</related>\n")
related = ''
cardsxml.write("</card>\n")
@ -535,13 +536,13 @@ def write_xml(mtgjson, code, name, releaseDate):
cardsxml.write("</cards>\n</cockatrice_carddatabase>")
if count > 0:
print 'XML Stats for ' + code
print 'Total cards: ' + str(count)
print ('XML Stats for ' + code)
print ('Total cards: ' + str(count))
if dfccount > 0:
print 'DFC: ' + str(dfccount)
print 'Newest: ' + str(newest)
print ('DFC: ' + str(dfccount))
print ('Newest: ' + str(newest))
else:
print 'Set ' + code + ' has no spoiled cards.'
print ('Set ' + code + ' has no spoiled cards.')
def write_combined_xml(mtgjson, setinfos):
@ -587,24 +588,24 @@ def write_combined_xml(mtgjson, setinfos):
newest = card["name"]
count += 1
name = card["name"]
if card.has_key("manaCost"):
if "manaCost" in card.keys():
manacost = card["manaCost"].replace('{', '').replace('}', '')
else:
manacost = ""
if card.has_key("power") or card.has_key("toughness"):
if "power" in card.keys() or "toughness" in card.keys():
if card["power"]:
pt = str(card["power"]) + "/" + str(card["toughness"])
else:
pt = 0
else:
pt = 0
if card.has_key("text"):
if "text" in card.keys():
text = card["text"]
else:
text = ""
cardcmc = str(card['cmc'])
cardtype = card["type"]
if card.has_key("names"):
if "names" in card.keys():
if "layout" in card:
if card["layout"] != 'split' and card["layout"] != 'aftermath':
if len(card["names"]) > 1:
@ -628,7 +629,7 @@ def write_combined_xml(mtgjson, setinfos):
text += "\n---\n" + cardb["text"]
name += " // " + cardb['name']
else:
print card["name"] + " has multiple names and no 'layout' key"
print (card["name"] + " has multiple names and no 'layout' key")
tablerow = "1"
if "Land" in cardtype:
@ -644,14 +645,14 @@ def write_combined_xml(mtgjson, setinfos):
if 'b' in card['number']:
if 'layout' in card:
if card['layout'] == 'split' or card['layout'] == 'aftermath':
# print "We're skipping " + card['name'] + " because it's the right side of a split card"
# print ("We're skipping " + card['name'] + " because it's the right side of a split card")
continue
cardsxml.write("<card>\n")
cardsxml.write("<name>" + name.encode('utf-8') + "</name>\n")
cardsxml.write("<name>" + name + "</name>\n")
cardsxml.write(
'<set rarity="' + card['rarity'] + '" picURL="' + card["url"] + '">' + setcode + '</set>\n')
if card.has_key('colors'):
if 'colors' in card.keys():
colorTranslate = {
"White": "W",
"Blue": "U",
@ -665,30 +666,30 @@ def write_combined_xml(mtgjson, setinfos):
if related:
# for relatedname in related:
cardsxml.write(
"<related>" + related.encode('utf-8') + "</related>\n")
"<related>" + related + "</related>\n")
related = ''
cardsxml.write(
"<manacost>" + manacost.encode('utf-8') + "</manacost>\n")
"<manacost>" + manacost + "</manacost>\n")
cardsxml.write("<cmc>" + cardcmc + "</cmc>\n")
cardsxml.write("<type>" + cardtype.encode('utf-8') + "</type>\n")
cardsxml.write("<type>" + cardtype + "</type>\n")
if pt:
cardsxml.write("<pt>" + pt + "</pt>\n")
cardsxml.write("<tablerow>" + tablerow + "</tablerow>\n")
cardsxml.write("<text>" + text.encode('utf-8') + "</text>\n")
cardsxml.write("<text>" + text + "</text>\n")
if name + ' enters the battlefield tapped' in text:
cardsxml.write("<cipt>1</cipt>\n")
if card.has_key('loyalty'):
if 'loyalty' in card.keys():
cardsxml.write(
"<loyalty>" + str(card['loyalty']) + "</loyalty>\n")
cardsxml.write("</card>\n")
cardsxml.write("</cards>\n</cockatrice_carddatabase>")
print 'XML COMBINED STATS'
print 'Total cards: ' + str(count)
print ('XML COMBINED STATS')
print ('Total cards: ' + str(count))
if dfccount > 0:
print 'DFC: ' + str(dfccount)
print 'Newest: ' + str(newest)
print ('DFC: ' + str(dfccount))
print ('Newest: ' + str(newest))
def pretty_xml(infile):
@ -757,7 +758,7 @@ def make_masterpieces(headers, AllSets, spoil):
matched = True
break
if not matched:
print "We couldn't find a card object to assign the data to for masterpiece " + masterpiece['name']
print ("We couldn't find a card object to assign the data to for masterpiece " + masterpiece['name'])
masterpieces2.append(masterpiece)
mpsjson = {
"name": headers['name'],
@ -779,11 +780,19 @@ def set_has_cards(setinfo, manual_cards, mtgjson):
if set == setinfo['code']:
return True
def download_file(url):
local_filename = url.split('/')[-1]
headers = {'user-agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko / 20071127 Firefox / 2.0.0.11'}
r = requests.get(url, stream=True, headers=headers)
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
return local_filename
def get_allsets():
headers = {'user-agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko / 20071127 Firefox / 2.0.0.11'}
json_file = requests.get('http://mtgjson.com/json/AllSets.json', headers=headers)
AllSets = json.loads(json_file.text)
file_location = download_file('https://mtgjson.com/json/AllSets.json.xz')
AllSets = json.loads(lzma.open(file_location).read())
return AllSets

4
verify_files.py
View File

@ -12,7 +12,7 @@ def load_file(input_file, lib_to_use):
output_file.append(doc)
return output_file
except Exception as ex:
print "Unable to load file: " + input_file + "\nException information:\n" + str(ex.args)
print ("Unable to load file: " + input_file + "\nException information:\n" + str(ex.args))
sys.exit("Unable to load file: " + input_file)
if __name__ == '__main__':
@ -21,4 +21,4 @@ if __name__ == '__main__':
card_corrections = load_file('cards_corrections.yml','yaml')
delete_cards = load_file('cards_delete.yml','yaml')
print "Pre-flight: All input files loaded successfully."
print ("Pre-flight: All input files loaded successfully.")

16
wizards_scraper.py
View File

@ -40,7 +40,7 @@ def scrape_fullspoil(url="http://magic.wizards.com/en/articles/archive/card-imag
cards.append(card)
cardcount += 1
fullspoil = {"cards": cards}
print "Spoil Gallery has " + str(cardcount) + " cards."
print ("Spoil Gallery has " + str(cardcount) + " cards.")
download_images(fullspoil['cards'], setinfo['code'])
fullspoil = get_rarities_by_symbol(fullspoil, setinfo['code'])
fullspoil = get_mana_symbols(fullspoil, setinfo['code'])
@ -90,9 +90,9 @@ def get_rarities_by_symbol(fullspoil, setcode):
card['rarity'] = color
if variance > highVariance:
# if a card isn't close to any of the colors, it's probably a planeswalker? make it mythic.
print card['name'], 'has high variance of', variance, ', closest rarity is', card['rarity']
print (card['name'], 'has high variance of', variance, ', closest rarity is', card['rarity'])
card['rarity'] = "Mythic Rare"
# print card['name'], '$', reds, greens, blues
# print (card['name'], '$', reds, greens, blues)
if symbolCount < 10:
setSymbol.save(
'images/' + card['name'].replace(' // ', '') + '.symbol.jpg')
@ -195,7 +195,7 @@ def get_mana_symbols(fullspoil={}, setcode="HOU"):
closestColor = color
if variance < 10:
# if card['name'] in ["Mirage Mirror", "Uncage the Menagerie", "Torment of Hailfire"]:
# print card['name'] + " " + str(reds) + " " + str(greens) + " " + str(blues)
# print (card['name'] + " " + str(reds) + " " + str(greens) + " " + str(blues))
if closestColor in ["2", "5"]:
twoVSfive = (
manaBox[0] + 1, manaBox[1] + 4, manaBox[2] - 5, manaBox[3] - 2)
@ -248,9 +248,9 @@ def smash_fullspoil(mtgjson, fullspoil):
if not match:
WOTC_only.append(fullspoil_card['name'])
if len(WOTC_only) > 0:
print "WOTC only cards: "
print WOTC_only
print different_keys
print ("WOTC only cards: ")
print (WOTC_only)
print (different_keys)
def download_images(mtgjson, setcode):
@ -264,6 +264,6 @@ def download_images(mtgjson, setcode):
if card['url']:
if os.path.isfile('images/' + setcode + '/' + card['name'].replace(' // ', '') + '.jpg'):
continue
# print 'Downloading ' + card['url'] + ' to images/' + setcode + '/' + card['name'].replace(' // ','') + '.jpg'
# print ('Downloading ' + card['url'] + ' to images/' + setcode + '/' + card['name'].replace(' // ','') + '.jpg')
requests.get(card['url'], 'images/' + setcode +
'/' + card['name'].replace(' // ', '') + '.jpg')