Return of Split Cards. MPS set starting. Sanitize Scryfall.

main.py

@@ -10,7 +10,10 @@ import json
 presets = {
     "isfullspoil": False, #when full spoil comes around, we only want to use WOTC images
     "includeMasterpieces": True, #if the set has masterpieces, let's get those too
-    "oldRSS": False #maybe MTGS hasn't updated their spoiler.rss but new cards have leaked
+    "oldRSS": False, #maybe MTGS hasn't updated their spoiler.rss but new cards have leaked
+    "split_cards": {
+        "Grind": "Dust"
+    }
 }
 
 with open('set_info.json') as data_file:
@@ -83,7 +86,7 @@ if __name__ == '__main__':
             mtgs = { "cards":[] }
         else:
             mtgs = spoilers.scrape_mtgs('http://www.mtgsalvation.com/spoilers.rss') #scrape mtgs rss feed
-            mtgs = spoilers.parse_mtgs(mtgs) #parse spoilers into mtgjson format
+            mtgs = spoilers.parse_mtgs(mtgs, [], [], [], presets['split_cards']) #parse spoilers into mtgjson format
         mtgs = spoilers.correct_cards(mtgs, manual_sets[setinfo['setname']]['cards'], card_corrections, delete_cards) #fix using the fixfiles
         scryfall = spoilers.get_scryfall('https://api.scryfall.com/cards/search?q=++e:' + setinfo['setname'].lower())
         mtgs = spoilers.get_image_urls(mtgs, presets['isfullspoil'], setinfo['setname'], setinfo['setlongname'], setinfo['setsize']) #get images

spoilers.py

@@ -489,6 +489,8 @@ def convert_scryfall(scryfall):
         else:
             card2['text'] = ''
         card2['url'] = card['image_uri']
+        if not 'type_line' in card:
+            card['type_line'] = 'Unknown'
         card2['type'] = card['type_line'].replace(u'—','-')
         cardtypes = card['type_line'].split(u' — ')[0].replace('Legendary ','').replace('Snow ','')\
         .replace('Elite ','').replace('Basic ','').replace('World ','').replace('Ongoing ','')
@@ -1037,20 +1039,24 @@ def make_allsets(AllSets, mtgjson, setname):
     AllSets[setname] = mtgjson
     return AllSets
 
-def scrape_masterpieces(url='http://www.mtgsalvation.com/spoilers/181-amonkhet-invocations', cardurl='http://www.mtgsalvation.com/cards/amonkhet-invocations/'):
+def scrape_masterpieces(url='http://www.mtgsalvation.com/spoilers/181-amonkhet-invocations', mtgscardurl='http://www.mtgsalvation.com/cards/amonkhet-invocations/'):
     page = requests.get(url)
     tree = html.fromstring(page.content)
     cards = []
     cardstree = tree.xpath('//*[contains(@class, "log-card")]')
     for child in cardstree:
-        #print child.text
-        cardpage = requests.get(cardurl + child.attrib['data-card-id'] + '-' + child.text.replace(' ','-'))
+        childurl = mtgscardurl + child.attrib['data-card-id'] + '-' + child.text.replace(' ','-')
+        cardpage = requests.get(childurl)
         tree = html.fromstring(cardpage.content)
         cardtree = tree.xpath('//img[contains(@class, "card-spoiler-image")]')
-        #print cardtree[0]
+        try:
+            cardurl = cardtree[0].attrib['src']
+        except:
+            cardurl = ''
+            pass
         card = {
             "name": child.text,
-            "url": cardtree[0].attrib['src']
+            "url": cardurl
         }
         cards.append(card)
     return cards