diff --git a/.travis/deploy.sh b/.travis/deploy.sh index 58413201..fff8cf50 100644 --- a/.travis/deploy.sh +++ b/.travis/deploy.sh @@ -73,7 +73,7 @@ else fi if [[ $ONLYDATECHANGE == true ]]; then git add -A . - git commit --allow-empty -m "Travis deploy to GitHub: ${SHA}" + git commit -m "Travis Deploy: ${SHA}" else echo "Only date in spoiler.xml changed, not committing fi diff --git a/main.py b/main.py index c20d4abd..1e479164 100644 --- a/main.py +++ b/main.py @@ -10,6 +10,7 @@ import io import sys import verify_files import requests +import yaml from lxml import etree presets = { @@ -66,9 +67,8 @@ def save_setjson(mtgs, filename): def save_errorlog(errorlog): - with open('out/errors.json', 'w') as outfile: - json.dump(errorlog, outfile, sort_keys=True, - indent=2, separators=(',', ': ')) + with open('out/errors.yml', 'w') as outfile: + yaml.safe_dump(errorlog, outfile, default_flow_style=False) def save_xml(xmlstring, outfile): @@ -127,10 +127,7 @@ if __name__ == '__main__': manual_cards = [] mtgs = spoilers.correct_cards( mtgs, manual_cards, card_corrections, delete_cards['delete']) # fix using the fixfiles - if not 'mythicCode' in setinfo: - setinfo['mythicCode'] = setinfo['code'] - mtgjson = spoilers.get_image_urls( - mtgs, presets['isfullspoil'], setinfo['code'], setinfo['mythicCode'], setinfo['name'], setinfo['size'], setinfo) # get images + mtgjson = spoilers.get_image_urls(mtgs, presets['isfullspoil'], setinfo) # get images if presets['scryfallOnly'] or 'scryfallOnly' in setinfo and setinfo['scryfallOnly']: scryfall = scryfall_scraper.get_scryfall( 'https://api.scryfall.com/cards/search?q=++e:' + setinfo['code'].lower()) @@ -184,7 +181,7 @@ if __name__ == '__main__': if presets['dumpErrors']: if errorlog != {}: print '//----- DUMPING ERROR LOG -----' - print json.dumps(errorlog, ensure_ascii=False, encoding='utf8', indent=2, sort_keys=True, separators=(',', ':')) + print yaml.safe_dump(errorlog, default_flow_style=False) print '//----- END ERROR LOG -----' else: print "No Detected Errors!" diff --git a/mtgs_scraper.py b/mtgs_scraper.py index 278bac6f..67845157 100644 --- a/mtgs_scraper.py +++ b/mtgs_scraper.py @@ -41,6 +41,9 @@ def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], rela print "Removing card scraped from MTGS RSS but not in their gallery: " + card['name'] cards.remove(card) + for card in cards: + card['name'] = card['name'].strip() + # if we didn't find any cards, let's bail out to prevent overwriting good data if len(cards) < 1: sys.exit("No cards found, exiting to prevent file overwrite") diff --git a/spoilers.py b/spoilers.py index a288e4b3..c9b4d466 100644 --- a/spoilers.py +++ b/spoilers.py @@ -302,19 +302,21 @@ def remove_corrected_errors(errorlog=[], card_corrections=[], print_fixed=False) return errorlog2 -def get_image_urls(mtgjson, isfullspoil, code, mythicCode, name, size=269, setinfo=False): +def get_image_urls(mtgjson, isfullspoil, setinfo=False): + if not 'mythicCode' in setinfo: + setinfo['mythicCode'] = setinfo['code'] IMAGES = 'http://magic.wizards.com/en/content/' + \ - name.lower().replace(' ', '-') + '-cards' + setinfo['name'].lower().replace(' ', '-') + '-cards' IMAGES2 = 'http://mythicspoiler.com/newspoilers.html' IMAGES3 = 'http://magic.wizards.com/en/articles/archive/card-image-gallery/' + \ - name.lower().replace('of', '').replace(' ', ' ').replace(' ', '-') + setinfo['name'].lower().replace('of', '').replace(' ', ' ').replace(' ', '-') text = requests.get(IMAGES).text text2 = requests.get(IMAGES2).text text3 = requests.get(IMAGES3).text wotcpattern = r'{}.*?' + mythicspoilerpattern = r' src="' + setinfo['mythicCode'].lower() + '/cards/{}.*?.jpg">' WOTC = [] for c in mtgjson['cards']: if 'names' in c: