From 319be1c26a9c6de1a186b80cae814455f85b00aa Mon Sep 17 00:00:00 2001 From: tritoch Date: Tue, 30 May 2017 09:23:36 -0500 Subject: [PATCH] Migrate Repo Bring in SpoilerSeason code --- .github/CONTRIBUTING.md | 17 + .gitignore | 20 +- .travis.yml | 26 ++ README.md | 37 +- cards_corrections.json | 18 + cards_delete.json | 1 + cards_manual.json | 61 +++ deploy.sh | 64 +++ deploy_key.enc | Bin 0 -> 3248 bytes main.py | 92 ++++ requirements.txt | 5 + set_info.json | 18 + spoilers.py | 960 ++++++++++++++++++++++++++++++++++++++++ 13 files changed, 1304 insertions(+), 15 deletions(-) create mode 100644 .github/CONTRIBUTING.md create mode 100644 .travis.yml create mode 100644 cards_corrections.json create mode 100644 cards_delete.json create mode 100644 cards_manual.json create mode 100644 deploy.sh create mode 100644 deploy_key.enc create mode 100644 main.py create mode 100644 requirements.txt create mode 100644 set_info.json create mode 100644 spoilers.py diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 00000000..869cb621 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,17 @@ +# Contributing to SpoilerSeason # +Thank you for your interest in contributing to SpoilerSeason! +This project is an attempt to create a central source for new Magic: the Gathering spoilers and provide data files for miscellaneous projects including our friends over at [Cockatrice](https://github.com/Cockatrice/Cockatrice) + +## How can I help? ## +SpoilerSeason grabs its data from many sources, but those sources often contain errors. If you just want to improve the card data and fix errors, you can start in the [errors.json](https://github.com/Cockatrice/Magic-Spoiler/blob/files/errors.json) file in the [files branch](https://github.com/Cockatrice/Magic-Spoiler/tree/files) or our [issue tracker.](https://github.com/Cockatrice/Magic-Spoiler/issues) +Once you've found an error, whether it be in the errors.json file or from using the data files, make sure that error hasn't already been fixed in the appropriate file on the [files branch.](https://github.com/Cockatrice/Magic-Spoiler/tree/files) If it's still present, let's get it fixed! +- If the error is with one of the fields in a card, check our [cards_corrections.json](https://github.com/Cockatrice/Magic-Spoiler/blob/master/cards_corrections.json) file. The syntax for this file is `"cardname": { "field to correct": "new value" }` If you're fixing the card name, you'd put the bad card name as `cardname` +- If the card shouldn't exist at all, check the [cards_delete.json](https://github.com/Cockatrice/Magic-Spoiler/blob/master/cards_delete.json) file. This file is just an array of cards to delete. Card name is case sensitive! +- If the card is a legitimate spoiler and it isn't showing up yet, you can manually add it. The file you want is [cards_manual.json](https://github.com/Cockatrice/Magic-Spoiler/blob/master/cards_manual.json) Make sure you link the spoil source in your Push Request. + +All Push requests for card fixes should have the name of the card being fixed and the type of fix (fix/correction, delete, or manual). In the details of the PR, you **MUST INCLUDE A VALID LINK** to the page the spoiler is located at. For minor fixes, a link to the card image is OK. And of course link the issue you're fixing if there is one! + +## Anything else? ## +If you notice errors, please file an [issue](https://github.com/Cockatrice/Magic-Spoiler/issues) + +Code improvement PRs are always welcome! diff --git a/.gitignore b/.gitignore index 7bbc71c0..d99b3558 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Project specific +AllSets.pre.json + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -20,7 +23,6 @@ lib64/ parts/ sdist/ var/ -wheels/ *.egg-info/ .installed.cfg *.egg @@ -43,7 +45,7 @@ htmlcov/ .cache nosetests.xml coverage.xml -*.cover +*,cover .hypothesis/ # Translations @@ -67,7 +69,7 @@ docs/_build/ # PyBuilder target/ -# Jupyter Notebook +# IPython Notebook .ipynb_checkpoints # pyenv @@ -76,26 +78,18 @@ target/ # celery beat schedule file celerybeat-schedule -# SageMath parsed files -*.sage.py - # dotenv .env # virtualenv -.venv venv/ ENV/ # Spyder project settings .spyderproject -.spyproject # Rope project settings .ropeproject -# mkdocs documentation -/site - -# mypy -.mypy_cache/ +# JetBrains +.idea diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..4e387289 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +language: python +install: + - pip install -r requirements.txt +script: bash ./deploy.sh +notifications: + email: false + webhooks: + urls: + - https://webhooks.gitter.im/e/7540dd1e121a1430fb8f + on_success: always # options: [always|never|change] default: always + on_failure: always # options: [always|never|change] default: always + on_start: never # options: [always|never|change] default: always +deploy: + provider: releases + api_key: + secure: p4QzePl5wafv0KSkxt9Y1TYhfv0h3p9gSUf+Km9MZiSVCNWpYY7hiWuKQ83RKX/IdqBzo1oJnitkk8ayCD26vlKH84Hf3aXIK4cbFApTNpO60FxnbvD1l+8OCyOq0IDul7jhszHlOHycjeSuMjn+Ok8qyc8MZSkMbFDgvt7Libr6fj8RtW6ozwM9yxZozkJ0lPNXfhYocDa8xgs5kCjWXy2ThFo08jjL4uMpmDoE0cAB9A6MZ9xma5NKuKF5JmecPP9qBjxOV8CcTjq5Chl9JkpBbcEmkD4FGsizTvo/0JjYOirQzEQ/i5b87X7XJ7AaGqkvVSp348t00n3fD4pEdEiLWPTFEnPV43VmAcqOfWtq1osrq0bEba0ktKyZzfT24Gf7jRMApT5/MbsM4tcOQUZTJufAiqCm44w84FqGWOie+r3pZ7jHyf8dlAexyEC802kvocpCARzY6EY+nXpyWDHL29DWajEMQyX32y9vqPem5p19JAwFFHhIMwzsjlCppEgcsxRIoE5S0qUO6wRUq8msC6co70Yl3UkMOssDbOpyqLNaUoCKlenbw3VaFgb5wqJzaPAhCoaQ7iLDZ6JQQkygrC20rTUN/xc34+U5gO0JPDOPUxUiuPfLc7/jwZaix2NJfzErUZnENjfNR8HLAhpSOXonMgMekJbMaYgxLi4= + file_glob: true + file: out/*.xml + skip_cleanup: true + overwrite: true + on: + tags: true +env: + global: + - ENCRYPTION_LABEL: 766e07f958d0 + - COMMIT_AUTHOR_EMAIL: you@example.com diff --git a/README.md b/README.md index 3573892b..5a19c404 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,35 @@ -# Magic-Spoiler -Tools for generating Cockatrice-compatible cards.xmls for unreleased magic cards +# SpoilerSeason # + +SpoilerSeason is a python script to scrape MTGS, Scryfall, and Wizards.com to compile a cockatrice-friendly XML file as well as json files. + +## Errors ## +Noticed an error? Check out our [Contributing file](https://github.com/Cockatrice/Magic-Spoiler/blob/master/.github/CONTRIBUTING.md) for information on how to help! + +## Running ## + +### Requirements ### + * Python 2.7 + * Python Modules: + requests==2.13.0 + feedparser + lxml + Pillow + datetime + +``` +pip install -r requirements.txt +``` + +### Usage ### + +``` +$> python main.py +``` + +Outputs out/{SETCODE}.xml, out/MPS\_{SETCODE}.xml, out/{SETCODE}.json, out/{MPS\_{SETCODE}.json + +errors are logged to out/errors.json + +Add the set xml file to your `customsets` folder for Cockatrice. + +When run by travis, uploads all files to [files branch](https://github.com/Cockatrice/Magic-Spoiler/tree/files) diff --git a/cards_corrections.json b/cards_corrections.json new file mode 100644 index 00000000..de07953c --- /dev/null +++ b/cards_corrections.json @@ -0,0 +1,18 @@ +{ + "Open into Wonder": { + "manaCost": "{X}{U}{U}" + }, + "By Force": { + "manaCost": "{X}{R}" + }, + "Nissa, Steward of Elements": { + "manaCost": "{X}{U}{G}", + "loyalty": "X" + }, + "Pull From Tomorrow": { + "name": "Pull from Tomorrow" + }, + "Tresspasser's Curse": { + "name": "Trespasser's Curse" + } +} \ No newline at end of file diff --git a/cards_delete.json b/cards_delete.json new file mode 100644 index 00000000..b4393b6c --- /dev/null +++ b/cards_delete.json @@ -0,0 +1 @@ +["~Sandwurm Convengence","Holaphel, Maw of Sands","Evolving Wilds"] diff --git a/cards_manual.json b/cards_manual.json new file mode 100644 index 00000000..7443059f --- /dev/null +++ b/cards_manual.json @@ -0,0 +1,61 @@ +{ + "meta": { + "instructions": "If you would like to add a card manually, use the 'blank card' template below.", + "instructions2": "Check example_card_details for more information", + "instructions3": "add the card to the 'cards' array below meta", + "example card (do not include this key, just the object)": { + "cmc": 7, + "colorIdentity": [ + "W", + "B", + "G" + ], + "colors": [ + "White", + "Black" + ], + "layout": "normal", + "manaCost": "{X}{5}{W}{B}", + "name": "Example Card", + "number": "55", + "power": "*", + "rarity": "Mythic Rare", + "subtypes": [ + "Zombie" + ], + "text": "Shadow, Flying, Horsemanship\nExample Card's power is equal to something.\n{G}, {T}: Unfloop target pig you control.", + "toughness": "2", + "type": "Legendary Creature - Zombie", + "types": [ + "Creature" + ] + }, + "example card details":{ + "meta": { + "values": "All fields (including loyalty) are string or array of strings except CMC which is int", + "required fields (all cards)": [ + "name", + "number", + "rarity", + "type", + "url" + ] + } + }, + "blank card (do not use this key, just the object) - remove unneeded keys": + { + "name": "", + "number": "", + "rarity": "", + "type": "", + "url": "", + "loyalty": "", + "cmc": 0, + "layout": "", + "power": "", + "toughness": "" + } + }, + "cards": [ + ] +} diff --git a/deploy.sh b/deploy.sh new file mode 100644 index 00000000..a5e8af97 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,64 @@ +#!/bin/bash +set -e # Exit with nonzero exit code if anything fails + +SOURCE_BRANCH="deploy-to-SpoilerSeasonFiles" +TARGET_BRANCH="files" + +function doCompile { + python main.py +} + +# Pull requests and commits to other branches shouldn't try to deploy, just build to verify +#if [ "$TRAVIS_PULL_REQUEST" != "false" -o "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then +# echo "Skipping deploy; just doing a build." +# doCompile +# exit 0 +#fi + +# Save some useful information +REPO=`git config remote.origin.url` +SSH_REPO=${REPO/https:\/\/github.com\//git@github.com:} +SHA=`git rev-parse --verify HEAD` + +# Clone the existing gh-pages for this repo into out/ +# Create a new empty branch if gh-pages doesn't exist yet (should only happen on first deply) +git clone $REPO out +cd out +git checkout $TARGET_BRANCH || git checkout --orphan $TARGET_BRANCH +cd .. + +# Clean out existing contents +rm -rf out/**/* || exit 0 + +# Run our compile script +doCompile + +# Now let's go have some fun with the cloned repo +cd out +ls +git config user.name "Travis CI" +git config user.email "$COMMIT_AUTHOR_EMAIL" + +# If there are no changes to the compiled out (e.g. this is a README update) then just bail. +#if git diff --quiet; then +# echo "No changes to the output on this push; exiting." +# exit 0 +#fi + +# Commit the "changes", i.e. the new version. +# The delta will show diffs between new and old versions. +git add -A . +git commit --allow-empty -m "Deploy to GitHub: ${SHA}" + +# Get the deploy key by using Travis's stored variables to decrypt deploy_key.enc +ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key" +ENCRYPTED_IV_VAR="encrypted_${ENCRYPTION_LABEL}_iv" +ENCRYPTED_KEY=${!ENCRYPTED_KEY_VAR} +ENCRYPTED_IV=${!ENCRYPTED_IV_VAR} +openssl aes-256-cbc -K $ENCRYPTED_KEY -iv $ENCRYPTED_IV -in ../deploy_key.enc -out ../deploy_key -d +chmod 600 ../deploy_key +eval `ssh-agent -s` +ssh-add ../deploy_key + +# Now that we're all set up, we can push. +git push $SSH_REPO $TARGET_BRANCH diff --git a/deploy_key.enc b/deploy_key.enc new file mode 100644 index 0000000000000000000000000000000000000000..a2ff1539b5ce69ee2d9798ac4e4ed2007974ee99 GIT binary patch literal 3248 zcmV;h3{Uf8p|1G$2Lm_Lc&$8=EyS_omcecY`SEK<(uwF9zLvH70AM2d!0`AknJdB{ zuk@f10|Ri$W>v)|bYk~ec*FBU&eI$E))5DT)Peeyz=NTF_TuS;kH?%Z)7R**fL5Jwc2kO zn-qpR7ciw@+x#&~bw$Lp{ut0MXa+a|Kp^!Y=9LK4wki1E(TXux)ozG@-iK4zL2ll} zb)A2*Wo>J=gAT3lvN*bhCuD;60iL(FwxU(5A0-VJW2?_uCtnus7x>CRvn&?`Q2J*$ zIF$%Jg=$9l>$$s z@@Ec&BsA_TCm%Je)@laq5+I>+5Y!G|LG(G){Rn@Bjdm+72;!@4A?0e*c_>aK`K&N@ zV7nUcItz7u7WSbdCRvBu*7uR-3oqrx;Wl(1{gjTrjnQt{z|aEwjG}PhJ1MP(3*-#ahDRgk1mFwVbsy4p)6X*p({9v7Ctn&zoM zq#SVWF1i6e0({ziSq{GL{n%B**cCw;bT=l8InX+_lFXu5$VWjy!|N%cg-2}Y`R5Zq zoXe;-&?a(=Jr-+YdMKsPEp0QJrf6*jO%`^Coy*!~G1<2{ZkQJT70^l6oU}k}y{F6_ zUIk!O=ZC_1Sx~W`WJ6tRj+F`xFNgckwL&9b7@3tbI2r3gfUW18!_g6!6q!pAC#zdG z`f<)&9Zkl=~6<^H!y`? zR8tRaGpiE%Do@vgY5uF!fU}`Yuh8;{j)9JL{=A@8nuTZ|VK$g(4$^3b;6a~c@3R`L z;l!Jz_+yGuC2Duoq2IX87PP?qxw--@wZJ0GtRF_d1cQN77QO@AkRK~R$Q>xSE{3b+ z!?|p-5rEJvcB6+ae8&`_)$1D87bJCJLRLLs9IU(mPc`(XBS$QR_7=m3|*!cwXU2g;n$LSLi~ks#fD z@nJ!*iZ}}D^52>+6p7)=!!jd)e4zel)OFp;F7icXK#hrn6q+Kw=ztcB03 zgY@BcY|nfBQwuHW*zzj20M3@R(zYZrd}qbCK9o;J+!*#!LSH;WNxBmz(d+AJ8~z((n>taIGs{T0`Ukm|LxQn+SY=Knb9$n9d$)z zdMHzV5gk`rboB8v%x!x}kXf5|Qrca6~Zt3?(8j>dix%{x10AZMKwPg>sw` zmJk#inzK$3JI=&mTw3v~tin@G(W|SP4OUGu@r{PMuiY1M$>uI<+pm6Bwhql*=gqip z_7cJyfMN<^S!Fy~qqq!K+zqtjF(({%E2FC0II4RDT2*ioit!-|#Ar8PKwZ>BU7MNC z^uYH=WR5N6*_^I`W|)fgQ7kRbkjxNFdei*pe5O z^cQUTN0OHRLy@nzcetR7M`P=K3W$_$tQ0bc4V(voncONCp3C5{hmZm}76~IPiJpS2 zVA!8Q+;g&Yc(vD#&0`M16Qi+;KltmF2oAT0Je$wu3b>~ z`Da=YEu5nedDX1a9!NL`av&{?}{vQivxoY7ayS%r!I)d>mLf5zhWZ!Xff z7uSPZd?jhP4RYm4_D)Ma^OT6@rc3(7il_jgUXH&_ubkQ#mC^#}bS8JXW^z)UPOOtF z%%;7}_}ubuFmS<8pdw?p(Im2>t-0d>m$G2BVV%2}qV;m+cBprb81)(@3o@0!q~Zk}&Hyf;oA)yymzHb*)J zwihMx)s9JGX7z8gT(KWq5!lT_n1QwE56FPS{9mQbA6YX}3ofzqofRL6BHuuT(~*5V zW!pf^F!T&}{-~LwjMblIX`;j8nzdg_<2F26o8#_!dI{bNvH-F&t>ikE%!hAq)l1lsSB?HcLeu-8 z#)BDpGWdn|)^UaXfF9ZGkue|zQ43#vd(mbIqtOU4;GEWU;O?ge z#;X&|+Wen~U^oGD(w&P`6UwrZHtHc9eyk27bYivVMUbA}Bgb0ZB{GipAXnjuGv`bV zpD=B{TcY!oRr_W0=WTPR_A4TYXC-VHjI|V%5TT0zTigvlxsIuq=Q=V+tyy1siCt|7 zTTkU>yiB6#oljBZ2qqe~g%)l3)i}OeJ)CAP#NN7+72q9J%8PpSJ-PxN3?v4P`fObu zWkDUe?5EJ4=;Ml1I~ddQcCNo#5|?1BO}`dC2aHInar4GJc|u`_UwJxXbm-|8bf=VZH!HCu{NaBerfqTH?g|7 zYcQfv#uZ>xEM4J?97mEvaR@MBA4()eNa%T!dv+gn=9Z2r@w;)A+hMQ`z&e8xrg~6ICBLw zn#h?U^Lbi~#L-WOxKd`%lPt#LY_kBnui5UC)(v}D=$IP_SskydWO9#KzCVm7Xj8;q zOA4EwMv}+}@?DKvaE*UEsaBeqo+EzLQ@3!3kmTS_v13hxs}hDx_`usMlo!(Xo_(Nm zmv>-XLH9tSd!oK@b>FtKjxH++NijC}A}xkG;WB7ZB>(Pi^n6N~i2Ly74U>$keq#PG zz+_mr7FlOI2|_3Dhn=IIc+F}K01Fl&FEP=h&J4PJ}Y$D`&VGR`&JCL-cg-LPf@|ur0f#KdHo2MtE zaPDSJN7poHT%vj8E{@D-`G(1I{lf|S=iZ3PXFcr@X}7}a(NQJGVs<4jqD}mjdnRWv zH;3jN3l)7s0By!s@=f@M%jSjwriiCx$oOk^xUYX?y*aa3V9#&j$5aPlAEx>jQ%})* z^7Vv^dKu#7Xi)n(;o6%zEo^jK;Sr_!lGOYL(hS5Cjyg2`KKQT0`0vwG*bRYsD6Nu2 ib!^jI(!h6hbxz&~g#-wLE`L?7FB}`lY;%gGBC8ZdI6n>m literal 0 HcmV?d00001 diff --git a/main.py b/main.py new file mode 100644 index 00000000..d1961934 --- /dev/null +++ b/main.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +import spoilers +import sys +#import configparser +import json +#import urllib + +presets = { + "isfullspoil": True, + "includeMasterpieces": True +} + +with open('set_info.json') as data_file: + setinfos = json.load(data_file) + +with open('cards_manual.json') as data_file: + manual_cards = json.load(data_file) + manual_cards = manual_cards['cards'] + +with open('cards_corrections.json') as data_file: + card_corrections = json.load(data_file) + +with open('cards_delete.json') as data_file: + delete_cards = json.load(data_file) + +errorlog = [] + +#TODO insert configparser to add config.ini file + +for argument in sys.argv: + for setinfo in setinfos: + if setinfo in argument.split("=")[0]: + setinfos[setinfo] = argument.split("=")[0] + for preset in presets: + if preset in argument.split("=")[0]: + presets[preset] = argument.split("=")[1] + +def save_allsets(AllSets): + print "Saving AllSets" + +def save_masterpieces(masterpieces): + with open('out/' + setinfos['masterpieces']['setname'] + '.json', 'w') as outfile: + json.dump(masterpieces, outfile, sort_keys=True, indent=2, separators=(',', ': ')) + +def save_setjson(mtgs): + with open('out/' + setinfos['setname'] + '.json', 'w') as outfile: + json.dump(mtgs, outfile, sort_keys=True, indent=2, separators=(',', ': ')) + +def save_errorlog(errorlog): + fixederrors = [] + unfixederrors = [] + for error in errorlog: + if 'fixed' in error: + fixederrors.append(error) + else: + unfixederrors.append(error) + errorlog = {"unfixed": unfixederrors, "fixed": fixederrors} + with open('out/errors.json', 'w') as outfile: + json.dump(errorlog, outfile, sort_keys=True, indent=2, separators=(',', ': ')) + +if __name__ == '__main__': + AllSets = spoilers.get_allsets() + mtgs = spoilers.scrape_mtgs('http://www.mtgsalvation.com/spoilers.rss') + mtgs = spoilers.parse_mtgs(mtgs) + mtgs = spoilers.correct_cards(mtgs, manual_cards, card_corrections, delete_cards) + #errorlog.append(temperror) + #scryfall = spoilers.get_scryfall('https://api.scryfall.com/cards/search?q=++e:' + setinfos['setname'].lower()) + mtgs = spoilers.get_image_urls(mtgs, presets['isfullspoil'], setinfos['setname'], setinfos['setlongname'], setinfos['setsize']) + #errorlog.append(temperror) + [mtgs, errors] = spoilers.errorcheck(mtgs) + errorlog += errors + spoilers.write_xml(mtgs, setinfos['setname'], setinfos['setlongname'], setinfos['setreleasedate']) + mtgs = spoilers.add_headers(mtgs, setinfos) + AllSets = spoilers.make_allsets(AllSets, mtgs, setinfos['setname']) + if 'masterpieces' in setinfos: + masterpieces = spoilers.make_masterpieces(setinfos['masterpieces'], AllSets, mtgs) + [masterpieces, errors] = spoilers.errorcheck(masterpieces) + errorlog += errors + #errorlog.append(temperror) + spoilers.write_xml(masterpieces, setinfos['masterpieces']['setname'], setinfos['masterpieces']['setlongname'], setinfos['masterpieces']['setreleasedate']) + AllSets = spoilers.make_allsets(AllSets, masterpieces, setinfos['masterpieces']['setname']) + save_masterpieces(masterpieces) + save_errorlog(errorlog) + save_allsets(AllSets) + save_setjson(mtgs) + +#outline +#set variables +#open files +#enable/disable features +#call spoilers +#save files diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..6baa507f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +requests==2.13.0 +feedparser +lxml +Pillow +datetime diff --git a/set_info.json b/set_info.json new file mode 100644 index 00000000..2db04f68 --- /dev/null +++ b/set_info.json @@ -0,0 +1,18 @@ +{ + "setname": "AKH", + "setlongname": "Amonkhet", + "blockname": "Amonkhet", + "setsize": 269, + "setreleasedate": "2017-04-28", + "settype": "expansion", + "masterpieces": { + "setname": "MPS_AKH", + "setlongname": "Masterpiece Series: Amonkhet Invocations", + "setreleasedate": "2017-04-28", + "alternativeNames": ["Amonkhet Invocations"], + "galleryURL": "http://magic.wizards.com/en/articles/archive/card-preview/masterpiece-series-amonkhet-invocations-2017-03-29", + "additionalCardNames": [], + "mtgsurl": "http://www.mtgsalvation.com/spoilers/181-amonkhet-invocations", + "mtgscardpath": "http://www.mtgsalvation.com/cards/amonkhet-invocations/" + } +} \ No newline at end of file diff --git a/spoilers.py b/spoilers.py new file mode 100644 index 00000000..ebf88a28 --- /dev/null +++ b/spoilers.py @@ -0,0 +1,960 @@ +# -*- coding: utf-8 -*- +import requests +import feedparser +import re +import sys +import os +import shutil +import time +from lxml import html +from lxml import etree +from PIL import Image +import datetime +import urllib +import json + +def scrape_mtgs(url): + return requests.get(url, headers={'Cache-Control':'no-cache', 'Pragma':'no-cache', 'Expires': 'Thu, 01 Jan 1970 00:00:00 GMT'}).text + +def parse_mtgs(mtgs, manual_cards=[], card_corrections=[], delete_cards=[], split_cards=[], related_cards=[]): + mtgs = mtgs.replace('utf-16','utf-8') + patterns = ['Name: (?P.*?)<', + 'Cost: (?P\d{0,2}[WUBRGC]*?)<', + 'Type: (?P.*?)<', + 'Pow/Tgh: (?P.*?)<', + 'Rules Text: (?P.*?)
.*?)<', + 'Set Number: #(?P.*?)/' + ] + d = feedparser.parse(mtgs) + + cards = [] + for entry in d.items()[5][1]: + card = dict(cost='',cmc='',img='',pow='',name='',rules='',type='', + color='', altname='', colorIdentity='', colorArray=[], colorIdentityArray=[], setnumber='', rarity='') + summary = entry['summary'] + for pattern in patterns: + match = re.search(pattern, summary, re.MULTILINE|re.DOTALL) + if match: + dg = match.groupdict() + card[dg.items()[0][0]] = dg.items()[0][1] + cards.append(card) + + #if we didn't find any cards, let's bail out to prevent overwriting good data + count = 0 + for card in cards: + count = count + 1 + if count < 1: + sys.exit("No cards found, exiting to prevent file overwrite") + + #for manual_card in manual_cards: + #initialize some keys + #manual_card['colorArray'] = [] + #manual_card['colorIdentityArray'] = [] + #manual_card['color'] = '' + #manual_card['colorIdentity'] = '' + #if not manual_card.has_key('rules'): + # manual_card['rules'] = '' + #if not manual_card.has_key('pow'): + # manual_card['pow'] = '' + #if not manual_card.has_key('setnumber'): + # manual_card['setnumber'] = '0' + #if not manual_card.has_key('type'): + # manual_card['type'] = '' + #see if this is a dupe + #and remove the spoiler version + #i trust my manual cards over their data + #for card in cards: + # if card['name'] == manual_card['name']: + # cards.remove(card) + #cards.append(manual_card) + + for card in cards: + card['name'] = card['name'].replace(''', '\'') + card['rules'] = card['rules'].replace(''', '\'') \ + .replace('<i>', '') \ + .replace('</i>', '') \ + .replace('"', '"') \ + .replace('blkocking', 'blocking')\ + .replace('&bull;','*')\ + .replace('comes into the','enters the')\ + .replace('threeor', 'three or')\ + .replace('[i]','')\ + .replace('[/i]','')\ + .replace('Lawlwss','Lawless')\ + .replace('Costner',"Counter") + card['type'] = card['type'].replace(' ',' ')\ + .replace('Crature', 'Creature') + if card['type'][-1] == ' ': + card['type'] = card['type'][:-1] + #if card['name'] in card_corrections: + # for correction in card_corrections[card['name']]: + # if correction != 'name': + # card[correction] = card_corrections[card['name']][correction] + # for correction in card_corrections[card['name']]: + # if correction == 'name': + # oldname = card['name'] + # card['name'] = card_corrections[oldname]['name'] + # card['rules'] = card['rules'].replace(oldname, card_corrections[oldname][correction]) + if 'cost' in card and len(card['cost']) > 0: + workingCMC = 0 + stripCost = card['cost'].replace('{','').replace('}','') + for manaSymbol in stripCost: + if manaSymbol.isdigit(): + workingCMC += int(manaSymbol) + elif not manaSymbol == 'X': + workingCMC += 1 + card['cmc'] = workingCMC + # figure out color + for c in 'WUBRG': + if c not in card['colorIdentity']: + if c in card['cost']: + card['color'] += c + card['colorIdentity'] += c + if (c + '}') in card['rules'] or (str.lower(c) + '}') in card['rules']: + if not (c in card['colorIdentity']): + card['colorIdentity'] += c + + cleanedcards = [] + + #let's remove any cards that are named in delete_cards array + for card in cards: + if not card['name'] in delete_cards: + cleanedcards.append(card) + cards = cleanedcards + + cardlist = [] + cardarray = [] + for card in cards: + dupe = False + for dupecheck in cardarray: + if dupecheck['name'] == card['name']: + dupe = True + if dupe == True: + continue + #if 'draft' in card['rules']: + # continue + for cid in card['colorIdentity']: + card['colorIdentityArray'].append(cid) + if 'W' in card['color']: + card['colorArray'].append('White') + if 'U' in card['color']: + card['colorArray'].append('Blue') + if 'B' in card['color']: + card['colorArray'].append('Black') + if 'R' in card['color']: + card['colorArray'].append('Red') + if 'G' in card['color']: + card['colorArray'].append('Green') + cardpower = '' + cardtoughness = '' + if len(card['pow'].split('/')) > 1: + cardpower = card['pow'].split('/')[0] + cardtoughness = card['pow'].split('/')[1] + cardnames = [] + cardnumber = card['setnumber'].lstrip('0') + if card['name'] in related_cards: + cardnames.append(card['name']) + cardnames.append(related_cards[card['name']]) + cardnumber += 'a' + card['layout'] = 'double-faced' + for namematch in related_cards: + if card['name'] == related_cards[namematch]: + card['layout'] = 'double-faced' + cardnames.append(namematch) + if not card['name'] in cardnames: + cardnames.append(card['name']) + cardnumber += 'b' + cardnames = [] + if card['name'] in split_cards: + cardnames.append(card['name']) + cardnames.append(split_cards[card['name']]) + cardnumber = cardnumber.replace('b','').replace('a','') + 'a' + card['layout'] = 'split' + for namematch in split_cards: + if card['name'] == split_cards[namematch]: + card['layout'] = 'split' + cardnames.append(namematch) + if not card['name'] in cardnames: + cardnames.append(card['name']) + cardnumber = cardnumber.replace('b','').replace('a','') + 'b' + if 'number' in card: + if 'b' in card['number'] or 'a' in card['number']: + if not 'layout' in card: + print card['name'] + " has a a/b number but no 'layout'" + + cardtypes = [] + if not '-' in card['type']: + card['type'] = card['type'].replace('instant','Instant').replace('sorcery','Sorcery').replace('creature','Creature') + cardtypes.append(card['type'].replace('instant','Instant')) + else: + cardtypes = card['type'].replace('Legendary ','').split(' - ')[0].split(' ')[:-1] + if '-' in card['type']: + subtype = card['type'].split(' - ')[1].strip() + #if u"—" in card['type']: + # subtype = card['type'].split(' — ')[1].strip() + if subtype: + subtypes = subtype.split(' ') + if card['cmc'] == '': + card['cmc'] = 0 + cardjson = {} + #cardjson["id"] = hashlib.sha1(setname + card['name'] + str(card['name']).lower()).hexdigest() + cardjson["cmc"] = card['cmc'] + cardjson["manaCost"] = card['cost'] + cardjson["name"] = card['name'] + cardjson["number"] = cardnumber + #not sure if mtgjson has a list of acceptable rarities, but my application does + #so we'll warn me but continue to write a non-standard rarity (timeshifted?) + #may force 'special' in the future + if card['rarity'] not in ['Mythic Rare','Rare','Uncommon','Common','Special']: + #errors.append({"name": card['name'], "key": "rarity", "value": card['rarity']}) + print card['name'] + ' has rarity = ' + card['rarity'] + if subtypes: + cardjson['subtypes'] = subtypes + cardjson["rarity"] = card['rarity'] + cardjson["text"] = card['rules'] + cardjson["type"] = card['type'] + cardjson["url"] = card['img'] + cardjson["types"] = cardtypes + #optional fields + if len(card['colorIdentityArray']) > 0: + cardjson["colorIdentity"] = card['colorIdentityArray'] + if len(card['colorArray']) > 0: + cardjson["colors"] = card['colorArray'] + if len(cardnames) > 1: + cardjson["names"] = cardnames + if cardpower or cardpower == '0': + cardjson["power"] = cardpower + cardjson["toughness"] = cardtoughness + if card.has_key('loyalty'): + cardjson["loyalty"] = card['loyalty'] + if card.has_key('layout'): + cardjson["layout"] = card['layout'] + + cardarray.append(cardjson) + + return {"cards": cardarray} + +def correct_cards(mtgjson, manual_cards=[], card_corrections=[], delete_cards=[]): + mtgjson2 = [] + for card in manual_cards: + if 'cmc' not in card: + workingCMC = 0 + stripCost = card['manaCost'].replace('{','').replace('}','') + for manaSymbol in stripCost: + if manaSymbol.isdigit(): + workingCMC += int(manaSymbol) + elif not manaSymbol == 'X': + workingCMC += 1 + if 'types' not in card: + card['types'] = [] + if '—' in card['type']: + workingTypes = card['type'].split('—')[0].strip() + else: + workingTypes = card['type'].split('-')[0].strip() + workingTypes.replace('Legendary ','').replace('Snow ','')\ + .replace('Elite ','').replace('Basic ','').replace('World ','').replace('Ongoing ','') + card['types'] += workingTypes.split(' ') + if 'subtypes' not in card: + if '—' in card['type']: + workingSubtypes = card['type'].split('—')[1].strip() + elif '-' in card['type']: + workingSubtypes = card['type'].split('-')[1].strip() + if workingSubtypes: + card['subtypes'] = workingSubtypes.split(' ') + colorMap = { + "W": "White", + "U": "Blue", + "B": "Black", + "R": "Red", + "G": "Green" + } + if 'manaCost' in card: + if 'text' in card and not 'Devoid' in card['text']: + for letter in card['manaCost']: + if not letter.isdigit() and not letter == 'X': + if 'colorIdentity' in card: + if not letter in card['colorIdentity']: + card['colorIdentity'] += letter + else: + card['colorIdentity'] = [letter] + if 'colors' in card: + if not colorMap[letter] in card['colors']: + card['colors'] += colorMap[letter] + else: + card['colors'] = [colorMap[letter]] + if 'text' in card: + for CID in colorMap: + if '{' + CID + '}' in card['text']: + if 'colorIdentity' in card: + if not CID in card['colorIdentity']: + card['colorIdentity'] += CID + else: + card['colorIdentity'] = [CID] + + for card in mtgjson['cards']: + isManual = False + for manualCard in manual_cards: + if card['name'] == manualCard['name']: + mtgjson2.append(manualCard) + isManual = True + if not isManual and not card['name'] in delete_cards: + mtgjson2.append(card) + mtgjson = {"cards": mtgjson2} + for card in mtgjson['cards']: + for cardCorrection in card_corrections: + if card['name'] == cardCorrection: + for correctionType in card_corrections[cardCorrection]: + if not correctionType == 'name': + card[correctionType] = card_corrections[cardCorrection][correctionType] + if 'name' in card_corrections[cardCorrection]: + card['name'] = card_corrections[cardCorrection]['name'] + return mtgjson + +def errorcheck(mtgjson): + errors = [] + for card in mtgjson['cards']: + for key in card: + if key == "": + errors.append({"name": card['name'], "key": key, "value": ""}) + requiredKeys = ['name','type'] + for requiredKey in requiredKeys: + if not requiredKey in card: + errors.append({"name": card['name'], "key": key, "missing": True}) + if 'text' in card: + #foo = 1 + card['text'] = card['text'].replace('','').replace('','').replace('','').replace(' 0: + if not 'manaCost' in card: + errors.append({"name": card['name'], "key": "manaCost", "value": "", "match": card['cmc']}) + else: + if 'manaCost' in card: + errors.append({"name": card['name'], "key": "manaCost", "oldvalue": card['manaCost'], "fixed": True}) + del card["manaCost"] + if 'colors' in card: + if not 'colorIdentity' in card: + if 'text' in card: + if not 'devoid' in card['text'].lower(): + errors.append({"name": card['name'], "key": "colorIdentity", "value": ""}) + else: + errors.append({"name": card['name'], "key": "colorIdentity", "value": ""}) + if 'colorIdentity' in card: + if not 'colors' in card: + #this one will false positive on emerge cards + if not 'Land' in card['type'] and not 'Artifact' in card['type'] and not 'Eldrazi' in card['type']: + if 'text' in card: + if not 'emerge' in card['text'].lower() and not 'devoid' in card['text'].lower(): + errors.append({"name": card['name'], "key": "colors", "value": ""}) + else: + errors.append({"name": card['name'], "key": "colors", "value": ""}) + #if not 'Land' in card['type'] and not 'Artifact' in card['type'] and not 'Eldrazi' in card['type']: + # errors.append({"name": card['name'], "key": "colors", "value": ""}) + if not 'url' in card: + errors.append({"name": card['name'], "key": "url", "value": ""}) + elif len(card['url']) < 10: + errors.append({"name": card['name'], "key": "url", "value": ""}) + if 'layout' in card: + if card['layout'] == 'split' or card['layout'] == 'meld' or card['layout'] == 'aftermath': + if not 'names' in card: + errors.append({"name": card['name'], "key": "names", "value": ""}) + if 'number' in card: + if not 'a' in card['number'] and not 'b' in card['number'] and not 'c' in card['number']: + errors.append({"name": card['name'], "key": "number", "value": card['number']}) + if not 'number' in card: + errors.append({"name": card['name'], "key": "number", "value": ""}) + if not 'types' in card: + errors.append({"name": card['name'], "key": "types", "value": ""}) + #print errors + return [mtgjson, errors] + +def get_scryfall(setUrl): + #getUrl = 'https://api.scryfall.com/cards/search?q=++e:' + #setUrl = getUrl + setname.lower() + setDone = False + scryfall = [] + + #firstPass = True + while setDone == False: + setcards = requests.get(setUrl) + setcards = setcards.json() + if setcards.has_key('data'): + #if firstPass: + # cards[set]["cards"] = [] + # firstPass = False + scryfall.append(setcards['data']) + #for setkey in mtgjson[set]: + # if 'card' not in setkey: + # if set != 'NMS': + # cards[set][setkey] = mtgjson[set][setkey] + else: + setDone = True + print setUrl + print setcards + print 'No data - ' + set + #noset.append(set) + time.sleep(.1) + if setcards.has_key('has_more'): + if setcards['has_more'] == True: + #print 'Going to extra page of ' + set + setUrl = setcards['next_page'] + else: + setDone = True + else: + setDone = True + #return scryfall + convert_scryfall(scryfall) + return scryfall + print + +def convert_scryfall(scryfall): + cards2 = [] + for card in scryfall: + card2 = {} + card2['cmc'] = int((card['converted_mana_cost']).split('.')[0]) + if card.has_key('mana_cost'): + card2['manaCost'] = card['mana_cost'].replace('{','').replace('}','') + else: + card2['manaCost'] = '' + card2['name'] = card['name'] + card2['number'] = card['collector_number'] + card2['rarity'] = card['rarity'].replace('mythic','mythic rare').title() + if card.has_key('oracle_text'): + card2['text'] = card['oracle_text'].replace(u"\u2022 ", u'*').replace(u"\u2014",'-').replace(u"\u2212","-") + else: + card2['text'] = '' + card2['url'] = card['image_uri'] + card2['type'] = card['type_line'].replace(u'—','-') + cardtypes = card['type_line'].split(u' — ')[0].replace('Legendary ','').replace('Snow ','')\ + .replace('Elite ','').replace('Basic ','').replace('World ','').replace('Ongoing ','') + cardtypes = cardtypes.split(' ') + if u' — ' in card['type_line']: + cardsubtypes = card['type_line'].split(u' — ')[1] + if ' ' in cardsubtypes: + card2['subtypes'] = cardsubtypes.split(' ') + else: + card2['subtypes'] = [cardsubtypes] + if 'Legendary' in card['type_line']: + if card2.has_key('supertypes'): + card2['supertypes'].append('Legendary') + else: + card2['supertypes'] = ['Legendary'] + if 'Snow' in card['type_line']: + if card2.has_key('supertypes'): + card2['supertypes'].append('Snow') + else: + card2['supertypes'] = ['Snow'] + if 'Elite' in card['type_line']: + if card2.has_key('supertypes'): + card2['supertypes'].append('Elite') + else: + card2['supertypes'] = ['Elite'] + if 'Basic' in card['type_line']: + if card2.has_key('supertypes'): + card2['supertypes'].append('Basic') + else: + card2['supertypes'] = ['Basic'] + if 'World' in card['type_line']: + if card2.has_key('supertypes'): + card2['supertypes'].append('World') + else: + card2['supertypes'] = ['World'] + if 'Ongoing' in card['type_line']: + if card2.has_key('supertypes'): + card2['supertypes'].append('Ongoing') + else: + card2['supertypes'] = ['Ongoing'] + card2['types'] = cardtypes + if card.has_key('color_identity'): + card2['colorIdentity'] = card['color_identity'] + if card.has_key('colors'): + if not card['colors'] == []: + card2['colors'] = [] + if 'W' in card['colors']: + card2['colors'].append("White") + if 'U' in card['colors']: + card2['colors'].append("Blue") + if 'B' in card['colors']: + card2['colors'].append("Black") + if 'R' in card['colors']: + card2['colors'].append("Red") + if 'G' in card['colors']: + card2['colors'].append("Green") + #card2['colors'] = card['colors'] + if card.has_key('all_parts'): + card2['names'] = [] + for partname in card['all_parts']: + card2['names'].append(partname['name']) + if card.has_key('power'): + card2['power'] = card['power'] + if card.has_key('toughness'): + card2['toughness'] = card['toughness'] + if card.has_key('layout'): + if card['layout'] != 'normal': + card2['layout'] = card['layout'] + if card.has_key('loyalty'): + card2['loyalty'] = card['loyalty'] + if card.has_key('artist'): + card2['artist'] = card['artist'] + #if card.has_key('source'): + # card2['source'] = card['source'] + #if card.has_key('rulings'): + # card2['rulings'] = card['rulings'] + if card.has_key('flavor_text'): + card2['flavor'] = card['flavor_text'] + if card.has_key('multiverse_id'): + card2['multiverseid'] = card['multiverse_id'] + + cards2['cards'].append(card2) + + return cards2 + print + +def scrape_fullspoil(url, showRarityColors=False, showFrameColors=False, manual_cards=[], delete_cards=[], split_cards=[]): + page = requests.get(url) + tree = html.fromstring(page.content) + cards = [] + cardtree = tree.xpath('//*[@id="content-detail-page-of-an-article"]') + for child in cardtree: + cardElements = child.xpath('//*/p/img') + cardcount = 0 + for cardElement in cardElements: + card = { + "name": cardElement.attrib['alt'].replace(u"\u2019",'\'').split(' /// ')[0], + "img": cardElement.attrib['src'] + } + card["url"] = card["img"] + card["cmc"] = 0 + card["manaCost"] = "" + card["type"] = "Land" + card["types"] = ["Land"] + card["text"] = "" + #card["colorIdentity"] = [""] + + if card['name'] in split_cards: + card["names"] = [card['name'], split_cards[card['name']]] + card["layout"] = "split" + notSplit = True + for backsplit in split_cards: + if card['name'] == split_cards[backsplit]: + notSplit = False + if notSplit and not card['name'] in delete_cards: + cards.append(card) + cardcount += 1 + print "Spoil Gallery has " + str(cardcount) + " cards." + #print mtgjson + #print cards + #return cards + get_rarities_by_symbol(fullspoil, showRarityColors) + get_colors_by_frame(fullspoil, showFrameColors) + return cards + +def get_rarities_by_symbol(fullspoil, split_cards=[]): + symbolPixels = (234, 215, 236, 218) + highVariance = 15 + colorAverages = { + "Common": [225, 224, 225], + "Uncommon": [194, 228, 240], + "Rare": [225, 201, 134], + "Mythic Rare": [249, 163, 15] + } + #symbolCount = 0 + for card in fullspoil: + cardImage = Image.open('images/' + card['name'] + '.png') + if card['name'] in split_cards: + setSymbol = cardImage.crop((234, 134, 236, 137)) + else: + setSymbol = cardImage.crop(symbolPixels) + cardHistogram = setSymbol.histogram() + reds = cardHistogram[0:256] + greens = cardHistogram[256:256 * 2] + blues = cardHistogram[256 * 2: 256 * 3] + reds = sum(i * w for i, w in enumerate(reds)) / sum(reds) + greens = sum(i * w for i, w in enumerate(greens)) / sum(greens) + blues = sum(i * w for i, w in enumerate(blues)) / sum(blues) + variance = 768 + for color in colorAverages: + colorVariance = 0 + colorVariance = colorVariance + abs(colorAverages[color][0] - reds) + colorVariance = colorVariance + abs(colorAverages[color][1] - greens) + colorVariance = colorVariance + abs(colorAverages[color][2] - blues) + if colorVariance < variance: + variance = colorVariance + card['rarity'] = color + if variance > highVariance: + # if a card isn't close to any of the colors, it's probably a planeswalker? make it mythic. + print card['name'], 'has high variance of', variance, ', closest rarity is', card['rarity'] + card['rarity'] = "Mythic Rare" + print card['name'], '$', reds, greens, blues + #if symbolCount < 10: + #setSymbol.save('images/' + card['name'] + '.symbol.jpg') + # symbolCount += 1 + return fullspoil + print + +def get_colors_by_frame(fullspoil, split_cards=[]): + framePixels = (20, 11, 76, 16) + highVariance = 10 + colorAverages = { + "White": [231,225,200], + "Blue": [103,193,230], + "Black": [58, 61, 54], + "Red": [221, 122, 101], + "Green": [118, 165, 131], + "Multicolor": [219, 200, 138], + "Artifact": [141, 165, 173], + "Colorless": [216, 197, 176], + } + #symbolCount = 0 + for card in fullspoil: + cardImage = Image.open('images/' + card['name'] + '.png') + if card['name'] in split_cards: + continue + #setSymbol = cardImage.crop((234, 134, 236, 137)) + #else: + cardColor = cardImage.crop(framePixels) + + cardHistogram = cardColor.histogram() + reds = cardHistogram[0:256] + greens = cardHistogram[256:256 * 2] + blues = cardHistogram[256 * 2: 256 * 3] + reds = sum(i * w for i, w in enumerate(reds)) / sum(reds) + greens = sum(i * w for i, w in enumerate(greens)) / sum(greens) + blues = sum(i * w for i, w in enumerate(blues)) / sum(blues) + variance = 768 + for color in colorAverages: + colorVariance = 0 + colorVariance = colorVariance + abs(colorAverages[color][0] - reds) + colorVariance = colorVariance + abs(colorAverages[color][1] - greens) + colorVariance = colorVariance + abs(colorAverages[color][2] - blues) + if colorVariance < variance: + variance = colorVariance + card['colors'] = [color] + if variance > highVariance: + # if a card isn't close to any of the colors, it's probably a planeswalker? make it mythic. + #print card['name'], 'has high variance of', variance, ', closest rarity is', card['color'] + print card['name'], '$ colors $', reds, greens, blues + #if 'Multicolor' in card['colors'] or 'Colorless' in card['colors'] or 'Artifact' in card['colors']: + # card['colors'] = [] + #if symbolCount < 10: + #cardColor.save('images/' + card['name'] + '.symbol.jpg') + # symbolCount += 1 + return fullspoil + +def get_image_urls(mtgjson, isfullspoil, setname, setlongname, setSize=269): + IMAGES = 'http://magic.wizards.com/en/content/' + setlongname.lower().replace(' ', '-') + '-cards' + IMAGES2 = 'http://mythicspoiler.com/newspoilers.html' + IMAGES3 = 'http://magic.wizards.com/en/articles/archive/card-image-gallery/' + setlongname.lower().replace(' ', '-') + + text = requests.get(IMAGES).text + text2 = requests.get(IMAGES2).text + text3 = requests.get(IMAGES3).text + wotcpattern = r'{}.*?' + for c in mtgjson['cards']: + match = re.search(wotcpattern.format(c['name'].replace('\'','’')), text, re.DOTALL) + if match: + c['url'] = match.groupdict()['img'] + else: + match3 = re.search(wotcpattern.format(c['name'].replace('\'','’')), text3, re.DOTALL) + if match3: + c['url'] = match3.groupdict()['img'] + else: + match2 = re.search(mythicspoilerpattern.format((c['name']).lower().replace(' ', '').replace(''', '').replace('-', '').replace('\'','').replace(',', '')), text2, re.DOTALL) + if match2 and not isfullspoil: + c['url'] = match2.group(0).replace(' src="', 'http://mythicspoiler.com/').replace('">', '') + pass + #if ('Creature' in c['type'] and not c.has_key('power')) or ('Vehicle' in c['type'] and not c.has_key('power')): + # print(c['name'] + ' is a creature w/o p/t img: ' + c['url']) + if len(str(c['url'])) < 10: + print(c['name'] + ' has no image.') + return mtgjson + +def write_xml(mtgjson, setname, setlongname, setreleasedate, split_cards=[]): + if not os.path.isdir('out/'): + os.makedirs('out/') + cardsxml = open('out/' + setname + '.xml', 'w+') + cardsxml.truncate() + count = 0 + dfccount = 0 + newest = '' + related = 0 + cardsxml.write("\n" + "\n" + "\n\n" + + setname + + "\n" + "" + + setlongname + + "\n" + "Expansion\n" + "" + + setreleasedate + + "\n" + "\n" + "\n" + "") + #print mtgjson + for card in mtgjson["cards"]: + for carda in split_cards: + if card["name"] == split_cards[carda]: + continue + if count == 0: + newest = card["name"] + count += 1 + name = card["name"] + if card.has_key("manaCost"): + manacost = card["manaCost"].replace('{', '').replace('}', '') + else: + manacost = "" + if card.has_key("power") or card.has_key("toughness"): + if card["power"]: + pt = str(card["power"]) + "/" + str(card["toughness"]) + else: + pt = 0 + else: + pt = 0 + if card.has_key("text"): + text = card["text"] + else: + text = "" + cardcmc = str(card['cmc']) + cardtype = card["type"] + if card.has_key("names"): + if "layout" in card: + if card["layout"] != 'split': + if len(card["names"]) > 1: + if card["names"][0] == card["name"]: + related = card["names"][1] + text += '\n\n(Related: ' + card["names"][1] + ')' + dfccount += 1 + elif card['names'][1] == card['name']: + related = card["names"][0] + text += '\n\n(Related: ' + card["names"][0] + ')' + else: + for carda in split_cards: + if card["name"] == carda: + cardb = split_cards[carda] + for jsoncard in mtgjson["cards"]: + if cardb == jsoncard["name"]: + cardtype += " // " + jsoncard["type"] + manacost += " // " + (jsoncard["manaCost"]).replace('{', '').replace('}', '') + cardcmc += " // " + str(jsoncard["cmc"]) + text += "\n---\n" + jsoncard["text"] + name += " // " + cardb + else: + print card["name"] + " has multiple names and no 'layout' key" + + + tablerow = "1" + if "Land" in cardtype: + tablerow = "0" + elif "Sorcery" in cardtype: + tablerow = "3" + elif "Instant" in cardtype: + tablerow = "3" + elif "Creature" in cardtype: + tablerow = "2" + + if 'number' in card: + if 'b' in card['number']: + if 'layout' in card: + if card['layout'] == 'split': + #print "We're skipping " + card['name'] + " because it's the right side of a split card" + continue + + cardsxml.write("\n") + cardsxml.write("" + name.encode('utf-8') + "\n") + cardsxml.write('' + setname + '\n') + cardsxml.write("" + manacost.encode('utf-8') + "\n") + cardsxml.write("" + cardcmc + "") + if card.has_key('colors'): + colorTranslate = { + "White": "W", + "Blue": "U", + "Black": "B", + "Red": "R", + "Green": "G" + } + for color in card['colors']: + cardsxml.write('' + colorTranslate[color] + '') + if name + ' enters the battlefield tapped' in text: + cardsxml.write("1") + cardsxml.write("" + cardtype.encode('utf-8') + "\n") + if pt: + cardsxml.write("" + pt + "\n") + if card.has_key('loyalty'): + cardsxml.write("" + str(card['loyalty']) + "") + cardsxml.write("" + tablerow + "\n") + cardsxml.write("" + text.encode('utf-8') + "\n") + if related: + # for relatedname in related: + cardsxml.write("" + related.encode('utf-8') + "\n") + related = '' + + cardsxml.write("\n") + + cardsxml.write("\n") + + #failing pretty xml code + #with open('out/' + setname + '.xml') as data_file: + # tree = etree.parse(data_file) + #root = tree.getroot() + #f = open('out/' + setname + '.xml', 'w') + #f.write(etree.tostring(root, pretty_print=True)) + #f.close() + + print 'XML STATS' + print 'Total cards: ' + str(count) + if dfccount > 0: + print 'DFC: ' + str(dfccount) + print 'Newest: ' + str(newest) + print 'Runtime: ' + str(datetime.datetime.today().strftime('%H:%M')) + ' on ' + str(datetime.date.today()) + +def make_allsets(AllSets, mtgjson, setname): + AllSets[setname] = mtgjson + return AllSets + +def scrape_masterpieces(url='http://www.mtgsalvation.com/spoilers/181-amonkhet-invocations', cardurl='http://www.mtgsalvation.com/cards/amonkhet-invocations/'): + page = requests.get(url) + tree = html.fromstring(page.content) + cards = [] + cardstree = tree.xpath('//*[contains(@class, "log-card")]') + for child in cardstree: + #print child.text + cardpage = requests.get(cardurl + child.attrib['data-card-id'] + '-' + child.text.replace(' ','-')) + tree = html.fromstring(cardpage.content) + cardtree = tree.xpath('//img[contains(@class, "card-spoiler-image")]') + #print cardtree[0] + card = { + "name": child.text, + "url": cardtree[0].attrib['src'] + } + cards.append(card) + return cards + +def make_masterpieces(headers, AllSets, spoil): + masterpieces = scrape_masterpieces(headers['mtgsurl'], headers['mtgscardpath']) + masterpieces2 = [] + for masterpiece in masterpieces: + matched = False + for set in AllSets: + if not matched: + for oldcard in AllSets[set]['cards']: + if oldcard['name'] == masterpiece['name'] and not matched: + mixcard = oldcard + mixcard['url'] = masterpiece['url'] + mixcard['rarity'] = 'Mythic Rare' + masterpieces2.append(mixcard) + matched = True + break + for spoilcard in spoil['cards']: + if not matched: + if spoilcard['name'] == masterpiece['name']: + mixcard = spoilcard + mixcard['rarity'] = 'Mythic Rare' + mixcard['url'] = masterpiece['url'] + masterpieces2.append(mixcard) + matched = True + break + if not matched: + print "We couldn't find a card object to assign the data to for masterpiece " + masterpiece['name'] + masterpieces2.append(masterpiece) + mpsjson = { + "name": headers['setlongname'], + "alternativeNames": headers['alternativeNames'], + "code": "MPS_AKH", + "releaseDate": headers['setreleasedate'], + "border": "black", + "type": "masterpiece", + "cards": masterpieces2 + } + return mpsjson + +def get_allsets(): + class MyOpener(urllib.FancyURLopener): + version = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko / 20071127 Firefox / 2.0.0.11' + + opener = MyOpener() + opener.retrieve('http://mtgjson.com/json/AllSets.json', 'AllSets.pre.json') + with open('AllSets.pre.json') as data_file: + AllSets = json.load(data_file) + return AllSets + +def add_headers(mtgjson, setinfos): + mtgjson2 = { + "block": setinfos['blockname'], + "border": "black", + "code": setinfos['setname'], + "magicCardsInfoCode": setinfos['setname'].lower(), + "name": setinfos['setlongname'], + "releaseDate": setinfos['setreleasedate'], + "type": setinfos['settype'], + "booster": [ + [ + "rare", + "mythic rare" + ], + "uncommon", + "uncommon", + "uncommon", + "common", + "common", + "common", + "common", + "common", + "common", + "common", + "common", + "common", + "common", + "land", + "marketing" + ], + "cards": mtgjson['cards'] + } + return mtgjson2