Remove cards from MTGS RSS that aren't in gallery (#112)

cards_delete.yml

@@ -7,8 +7,4 @@
 #   - " Test Card "
 
 delete:
-  - Sunset Pyramid (TO DELETE)
-  - Thoughtseize
-  - "Endless "
-  - Crumbling Necropolis
-  - Imaginary Thr
+  - Cannot be empty

main.py

@@ -124,7 +124,7 @@ if __name__ == '__main__':
         else:
             mtgs = mtgs_scraper.scrape_mtgs(
                 'http://www.mtgsalvation.com/spoilers.rss')  # scrape mtgs rss feed
-            mtgs = mtgs_scraper.parse_mtgs(mtgs)  # parse spoilers into mtgjson format
+            mtgs = mtgs_scraper.parse_mtgs(mtgs, setinfo=setinfo)  # parse spoilers into mtgjson format
         mtgs = spoilers.correct_cards(
             mtgs, manual_sets[setinfo['code']], card_corrections, delete_cards['delete'])  # fix using the fixfiles
         mtgjson = spoilers.get_image_urls(

mtgs_scraper.py

@@ -11,7 +11,7 @@ def scrape_mtgs(url):
     return requests.get(url, headers={'Cache-Control': 'no-cache', 'Pragma': 'no-cache', 'Expires': 'Thu, 01 Jan 1970 00:00:00 GMT'}).text
 
 
-def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], related_cards=[]):
+def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], related_cards=[], setinfo={"mtgsurl": ""}):
     mtgs = mtgs.replace('utf-16', 'utf-8')
     patterns = ['<b>Name:</b> <b>(?P<name>.*?)<',
                 'Cost: (?P<cost>[X]*\d{0,2}[XWUBRGC]*?)<',
@@ -35,6 +35,12 @@ def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], rela
                 card[dg.items()[0][0]] = dg.items()[0][1]
         cards.append(card)
 
+    gallery_list = list_mtgs_gallery(setinfo['mtgsurl'])
+    for card in cards:
+        if card['name'] not in gallery_list:
+            print "Removing card scraped from MTGS RSS but not in their gallery: " + card['name']
+            cards.remove(card)
+
     # if we didn't find any cards, let's bail out to prevent overwriting good data
     count = 0
     for card in cards:
@@ -255,3 +261,13 @@ def scrape_mtgs_images(url='http://www.mtgsalvation.com/spoilers/183-hour-of-dev
         }
         time.sleep(.2)
     return cards
+
+
+def list_mtgs_gallery(url=''):
+    page = requests.get(url)
+    tree = html.fromstring(page.content)
+    cards = []
+    cardstree = tree.xpath('//*[contains(@class, "log-card")]')
+    for child in cardstree:
+        cards.append(child.text)
+    return cards