diff --git a/main.py b/main.py index 3c2abb64..ed4791b9 100644 --- a/main.py +++ b/main.py @@ -10,7 +10,10 @@ import json presets = { "isfullspoil": False, #when full spoil comes around, we only want to use WOTC images "includeMasterpieces": True, #if the set has masterpieces, let's get those too - "oldRSS": False #maybe MTGS hasn't updated their spoiler.rss but new cards have leaked + "oldRSS": False, #maybe MTGS hasn't updated their spoiler.rss but new cards have leaked + "split_cards": { + "Grind": "Dust" + } } with open('set_info.json') as data_file: @@ -83,7 +86,7 @@ if __name__ == '__main__': mtgs = { "cards":[] } else: mtgs = spoilers.scrape_mtgs('http://www.mtgsalvation.com/spoilers.rss') #scrape mtgs rss feed - mtgs = spoilers.parse_mtgs(mtgs) #parse spoilers into mtgjson format + mtgs = spoilers.parse_mtgs(mtgs, [], [], [], presets['split_cards']) #parse spoilers into mtgjson format mtgs = spoilers.correct_cards(mtgs, manual_sets[setinfo['setname']]['cards'], card_corrections, delete_cards) #fix using the fixfiles scryfall = spoilers.get_scryfall('https://api.scryfall.com/cards/search?q=++e:' + setinfo['setname'].lower()) mtgs = spoilers.get_image_urls(mtgs, presets['isfullspoil'], setinfo['setname'], setinfo['setlongname'], setinfo['setsize']) #get images diff --git a/spoilers.py b/spoilers.py index d0d5c5f5..9b55f5c7 100644 --- a/spoilers.py +++ b/spoilers.py @@ -489,6 +489,8 @@ def convert_scryfall(scryfall): else: card2['text'] = '' card2['url'] = card['image_uri'] + if not 'type_line' in card: + card['type_line'] = 'Unknown' card2['type'] = card['type_line'].replace(u'—','-') cardtypes = card['type_line'].split(u' — ')[0].replace('Legendary ','').replace('Snow ','')\ .replace('Elite ','').replace('Basic ','').replace('World ','').replace('Ongoing ','') @@ -1037,20 +1039,24 @@ def make_allsets(AllSets, mtgjson, setname): AllSets[setname] = mtgjson return AllSets -def scrape_masterpieces(url='http://www.mtgsalvation.com/spoilers/181-amonkhet-invocations', cardurl='http://www.mtgsalvation.com/cards/amonkhet-invocations/'): +def scrape_masterpieces(url='http://www.mtgsalvation.com/spoilers/181-amonkhet-invocations', mtgscardurl='http://www.mtgsalvation.com/cards/amonkhet-invocations/'): page = requests.get(url) tree = html.fromstring(page.content) cards = [] cardstree = tree.xpath('//*[contains(@class, "log-card")]') for child in cardstree: - #print child.text - cardpage = requests.get(cardurl + child.attrib['data-card-id'] + '-' + child.text.replace(' ','-')) + childurl = mtgscardurl + child.attrib['data-card-id'] + '-' + child.text.replace(' ','-') + cardpage = requests.get(childurl) tree = html.fromstring(cardpage.content) cardtree = tree.xpath('//img[contains(@class, "card-spoiler-image")]') - #print cardtree[0] + try: + cardurl = cardtree[0].attrib['src'] + except: + cardurl = '' + pass card = { "name": child.text, - "url": cardtree[0].attrib['src'] + "url": cardurl } cards.append(card) return cards