diff --git a/cards_delete.yml b/cards_delete.yml index 68292d26..2c982b2d 100644 --- a/cards_delete.yml +++ b/cards_delete.yml @@ -7,8 +7,4 @@ # - " Test Card " delete: - - Sunset Pyramid (TO DELETE) - - Thoughtseize - - "Endless " - - Crumbling Necropolis - - Imaginary Thr \ No newline at end of file + - Cannot be empty \ No newline at end of file diff --git a/main.py b/main.py index ae0ba3ac..82353ed2 100644 --- a/main.py +++ b/main.py @@ -124,7 +124,7 @@ if __name__ == '__main__': else: mtgs = mtgs_scraper.scrape_mtgs( 'http://www.mtgsalvation.com/spoilers.rss') # scrape mtgs rss feed - mtgs = mtgs_scraper.parse_mtgs(mtgs) # parse spoilers into mtgjson format + mtgs = mtgs_scraper.parse_mtgs(mtgs, setinfo=setinfo) # parse spoilers into mtgjson format mtgs = spoilers.correct_cards( mtgs, manual_sets[setinfo['code']], card_corrections, delete_cards['delete']) # fix using the fixfiles mtgjson = spoilers.get_image_urls( diff --git a/mtgs_scraper.py b/mtgs_scraper.py index 03b86527..2a6481c2 100644 --- a/mtgs_scraper.py +++ b/mtgs_scraper.py @@ -11,7 +11,7 @@ def scrape_mtgs(url): return requests.get(url, headers={'Cache-Control': 'no-cache', 'Pragma': 'no-cache', 'Expires': 'Thu, 01 Jan 1970 00:00:00 GMT'}).text -def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], related_cards=[]): +def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], related_cards=[], setinfo={"mtgsurl": ""}): mtgs = mtgs.replace('utf-16', 'utf-8') patterns = ['Name: (?P.*?)<', 'Cost: (?P[X]*\d{0,2}[XWUBRGC]*?)<', @@ -35,6 +35,12 @@ def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], rela card[dg.items()[0][0]] = dg.items()[0][1] cards.append(card) + gallery_list = list_mtgs_gallery(setinfo['mtgsurl']) + for card in cards: + if card['name'] not in gallery_list: + print "Removing card scraped from MTGS RSS but not in their gallery: " + card['name'] + cards.remove(card) + # if we didn't find any cards, let's bail out to prevent overwriting good data count = 0 for card in cards: @@ -255,3 +261,13 @@ def scrape_mtgs_images(url='http://www.mtgsalvation.com/spoilers/183-hour-of-dev } time.sleep(.2) return cards + + +def list_mtgs_gallery(url=''): + page = requests.get(url) + tree = html.fromstring(page.content) + cards = [] + cardstree = tree.xpath('//*[contains(@class, "log-card")]') + for child in cardstree: + cards.append(child.text) + return cards \ No newline at end of file