diff --git a/cards_delete.yml b/cards_delete.yml
index 68292d26..2c982b2d 100644
--- a/cards_delete.yml
+++ b/cards_delete.yml
@@ -7,8 +7,4 @@
# - " Test Card "
delete:
- - Sunset Pyramid (TO DELETE)
- - Thoughtseize
- - "Endless "
- - Crumbling Necropolis
- - Imaginary Thr
\ No newline at end of file
+ - Cannot be empty
\ No newline at end of file
diff --git a/main.py b/main.py
index ae0ba3ac..82353ed2 100644
--- a/main.py
+++ b/main.py
@@ -124,7 +124,7 @@ if __name__ == '__main__':
else:
mtgs = mtgs_scraper.scrape_mtgs(
'http://www.mtgsalvation.com/spoilers.rss') # scrape mtgs rss feed
- mtgs = mtgs_scraper.parse_mtgs(mtgs) # parse spoilers into mtgjson format
+ mtgs = mtgs_scraper.parse_mtgs(mtgs, setinfo=setinfo) # parse spoilers into mtgjson format
mtgs = spoilers.correct_cards(
mtgs, manual_sets[setinfo['code']], card_corrections, delete_cards['delete']) # fix using the fixfiles
mtgjson = spoilers.get_image_urls(
diff --git a/mtgs_scraper.py b/mtgs_scraper.py
index 03b86527..2a6481c2 100644
--- a/mtgs_scraper.py
+++ b/mtgs_scraper.py
@@ -11,7 +11,7 @@ def scrape_mtgs(url):
return requests.get(url, headers={'Cache-Control': 'no-cache', 'Pragma': 'no-cache', 'Expires': 'Thu, 01 Jan 1970 00:00:00 GMT'}).text
-def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], related_cards=[]):
+def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], related_cards=[], setinfo={"mtgsurl": ""}):
mtgs = mtgs.replace('utf-16', 'utf-8')
patterns = ['Name: (?P.*?)<',
'Cost: (?P[X]*\d{0,2}[XWUBRGC]*?)<',
@@ -35,6 +35,12 @@ def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], rela
card[dg.items()[0][0]] = dg.items()[0][1]
cards.append(card)
+ gallery_list = list_mtgs_gallery(setinfo['mtgsurl'])
+ for card in cards:
+ if card['name'] not in gallery_list:
+ print "Removing card scraped from MTGS RSS but not in their gallery: " + card['name']
+ cards.remove(card)
+
# if we didn't find any cards, let's bail out to prevent overwriting good data
count = 0
for card in cards:
@@ -255,3 +261,13 @@ def scrape_mtgs_images(url='http://www.mtgsalvation.com/spoilers/183-hour-of-dev
}
time.sleep(.2)
return cards
+
+
+def list_mtgs_gallery(url=''):
+ page = requests.get(url)
+ tree = html.fromstring(page.content)
+ cards = []
+ cardstree = tree.xpath('//*[contains(@class, "log-card")]')
+ for child in cardstree:
+ cards.append(child.text)
+ return cards
\ No newline at end of file