From 9693dad6286522d165f38c84dca69591f8b0230b Mon Sep 17 00:00:00 2001 From: dev-id Date: Sat, 2 Sep 2017 21:21:29 -0500 Subject: [PATCH 1/4] Refactor get_image_urls function to use only `setinfo` as input. --- main.py | 5 +---- spoilers.py | 10 ++++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index c20d4abd..2ce49357 100644 --- a/main.py +++ b/main.py @@ -127,10 +127,7 @@ if __name__ == '__main__': manual_cards = [] mtgs = spoilers.correct_cards( mtgs, manual_cards, card_corrections, delete_cards['delete']) # fix using the fixfiles - if not 'mythicCode' in setinfo: - setinfo['mythicCode'] = setinfo['code'] - mtgjson = spoilers.get_image_urls( - mtgs, presets['isfullspoil'], setinfo['code'], setinfo['mythicCode'], setinfo['name'], setinfo['size'], setinfo) # get images + mtgjson = spoilers.get_image_urls(mtgs, presets['isfullspoil'], setinfo) # get images if presets['scryfallOnly'] or 'scryfallOnly' in setinfo and setinfo['scryfallOnly']: scryfall = scryfall_scraper.get_scryfall( 'https://api.scryfall.com/cards/search?q=++e:' + setinfo['code'].lower()) diff --git a/spoilers.py b/spoilers.py index a288e4b3..c9b4d466 100644 --- a/spoilers.py +++ b/spoilers.py @@ -302,19 +302,21 @@ def remove_corrected_errors(errorlog=[], card_corrections=[], print_fixed=False) return errorlog2 -def get_image_urls(mtgjson, isfullspoil, code, mythicCode, name, size=269, setinfo=False): +def get_image_urls(mtgjson, isfullspoil, setinfo=False): + if not 'mythicCode' in setinfo: + setinfo['mythicCode'] = setinfo['code'] IMAGES = 'http://magic.wizards.com/en/content/' + \ - name.lower().replace(' ', '-') + '-cards' + setinfo['name'].lower().replace(' ', '-') + '-cards' IMAGES2 = 'http://mythicspoiler.com/newspoilers.html' IMAGES3 = 'http://magic.wizards.com/en/articles/archive/card-image-gallery/' + \ - name.lower().replace('of', '').replace(' ', ' ').replace(' ', '-') + setinfo['name'].lower().replace('of', '').replace(' ', ' ').replace(' ', '-') text = requests.get(IMAGES).text text2 = requests.get(IMAGES2).text text3 = requests.get(IMAGES3).text wotcpattern = r'{}.*?' + mythicspoilerpattern = r' src="' + setinfo['mythicCode'].lower() + '/cards/{}.*?.jpg">' WOTC = [] for c in mtgjson['cards']: if 'names' in c: From 00dd79bf14018463aa7be7ae075dee98688a6ce3 Mon Sep 17 00:00:00 2001 From: dev-id Date: Sat, 2 Sep 2017 21:51:04 -0500 Subject: [PATCH 2/4] Strip leading/trailing spaces from card names. May prevent certain image sources from working (mtgs?) --- mtgs_scraper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mtgs_scraper.py b/mtgs_scraper.py index 278bac6f..67845157 100644 --- a/mtgs_scraper.py +++ b/mtgs_scraper.py @@ -41,6 +41,9 @@ def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], rela print "Removing card scraped from MTGS RSS but not in their gallery: " + card['name'] cards.remove(card) + for card in cards: + card['name'] = card['name'].strip() + # if we didn't find any cards, let's bail out to prevent overwriting good data if len(cards) < 1: sys.exit("No cards found, exiting to prevent file overwrite") From 9d00b4f05a36df8d92a394e11fe6c0f7ed8c633e Mon Sep 17 00:00:00 2001 From: Dave Date: Mon, 4 Sep 2017 10:45:50 -0500 Subject: [PATCH 3/4] Error log to YAML (#137) * Error log to YAML * Also dump debug print to YAML --- main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index c20d4abd..b269f1ec 100644 --- a/main.py +++ b/main.py @@ -10,6 +10,7 @@ import io import sys import verify_files import requests +import yaml from lxml import etree presets = { @@ -66,9 +67,8 @@ def save_setjson(mtgs, filename): def save_errorlog(errorlog): - with open('out/errors.json', 'w') as outfile: - json.dump(errorlog, outfile, sort_keys=True, - indent=2, separators=(',', ': ')) + with open('out/errors.yml', 'w') as outfile: + yaml.safe_dump(errorlog, outfile, default_flow_style=False) def save_xml(xmlstring, outfile): @@ -184,7 +184,7 @@ if __name__ == '__main__': if presets['dumpErrors']: if errorlog != {}: print '//----- DUMPING ERROR LOG -----' - print json.dumps(errorlog, ensure_ascii=False, encoding='utf8', indent=2, sort_keys=True, separators=(',', ':')) + print yaml.safe_dump(errorlog, default_flow_style=False) print '//----- END ERROR LOG -----' else: print "No Detected Errors!" From 01fcaec7c9e138003632be918b1025d40d557094 Mon Sep 17 00:00:00 2001 From: Dave Date: Mon, 4 Sep 2017 13:49:53 -0500 Subject: [PATCH 4/4] No empty commits from Travis --- .travis/deploy.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis/deploy.sh b/.travis/deploy.sh index ae6ff8cd..a597b60c 100644 --- a/.travis/deploy.sh +++ b/.travis/deploy.sh @@ -60,7 +60,8 @@ git config user.email "$COMMIT_AUTHOR_EMAIL" # Commit the "changes", i.e. the new version. # The delta will show diffs between new and old versions. git add -A . -git commit --allow-empty -m "Deploy to GitHub: ${SHA}" +#git commit --allow-empty -m "Deploy to GitHub: ${SHA}" +git commit -m "Travis Deploy ${SHA}" # Get the deploy key by using Travis's stored variables to decrypt deploy_key.enc ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key"