diff --git a/.ci/deploy.sh b/.ci/deploy.sh deleted file mode 100755 index 58a7aaea..00000000 --- a/.ci/deploy.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -e # Exit with nonzero exit code if anything fails - -function doCompile { - echo "Running script..." - python3 -m magic_spoiler -} - -# Pull requests and commits to other branches shouldn't try to deploy, just build to verify -if [[ ! -d $OUTPUT_PATH ]]; then - mkdir "$OUTPUT_PATH" - echo "Skipping deploy; just doing a build." - # Run our compile script and let user know in logs - doCompile - exit 0 -fi - -# Run our compile script and exit gracefully if there are no updates -if ! doCompile; then - echo "::warning::No changes!" - exit 0 -fi - -cd "$OUTPUT_PATH" -git config user.name github-actions -git config user.email github-actions@github.com - -# We don't want the AllSets... waste of space -git add -A . -git commit -m "Deploy: ${GITHUB_SHA}" - -# push using built-in token -git push diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b1ba14bb..52d89d0f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,6 +1,7 @@ name: Deploy on: + workflow_dispatch: push: branches: - master @@ -12,14 +13,19 @@ on: paths-ignore: - '**.md' schedule: - # every 6 hours = 4 times a day - - cron: '0 */6 * * *' + # Every 8 hours = 3 times a day + - cron: '0 */8 * * *' jobs: deploy: + # Do not run the scheduled workflow on forks + if: github.event_name != 'schedule' || github.repository_owner == 'Cockatrice' + + name: Check for new spoiler runs-on: ubuntu-latest env: + DEPLOY: ${{github.ref == 'refs/heads/master'}} OUTPUT_PATH: out steps: @@ -27,7 +33,8 @@ jobs: uses: actions/checkout@v2 - name: Checkout output branch - if: github.ref == 'refs/heads/master' + # Run only when triggered from master + if: env.DEPLOY == 'true' uses: actions/checkout@v2 with: ref: files @@ -42,6 +49,30 @@ jobs: python3 -m pip install --upgrade pip setuptools python3 -m pip install --requirement requirements.txt - - name: Deploy + - name: Run script + id: run shell: bash - run: ./.ci/deploy.sh + run: python3 -m magic_spoiler + + - name: Upload artifacts + # Run only when triggered from a PR + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v2 + with: + name: spoiler-output + path: ${{github.workspace}}/${{env.OUTPUT_PATH}} + if-no-files-found: error + + - name: Deploy changes + # Run only when triggered from master and changes are available + if: env.DEPLOY == 'true' && steps.run.outputs.deploy == 'true' + shell: bash + working-directory: ${{env.OUTPUT_PATH}} + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git add -A . + git commit -m "Deploy: $GITHUB_SHA" + git push + deploy_commit=`git rev-parse HEAD` + echo "::notice title=New data uploaded::See deployment: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$deploy_commit" diff --git a/.gitignore b/.gitignore index f12822ff..0fa3782e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ # Project specific out/ AllSets.pre.json -deploy_key.enc # Byte-compiled / optimized / DLL files __pycache__/ @@ -86,6 +85,9 @@ celerybeat-schedule # virtualenv venv/ ENV/ +bin/ +include/ +pyvenv.cfg # Spyder project settings .spyderproject @@ -100,4 +102,4 @@ ENV/ .DS_Store *.sqlite -.*_cache \ No newline at end of file +.*_cache diff --git a/README.md b/README.md index 6f9ca614..6632c72a 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,21 @@ # Magic-Spoiler [![Discord](https://img.shields.io/discord/314987288398659595?label=Discord&logo=discord&logoColor=white)](https://discord.gg/3Z9yzmA) [![Gitter Chat](https://img.shields.io/gitter/room/Cockatrice/Magic-Spoiler)](https://gitter.im/Cockatrice/Magic-Spoiler) # -Magic-Spoiler is a Python script to scrape [Scryfall](https://scryfall.com) to compile XML files (Cockatrice formatted) and application-ready json files (mtgjson formatted) with information about spoiled cards from upcoming sets. +Magic-Spoiler is a Python script to query the [Scryfall](https://scryfall.com) API to compile XML files (Cockatrice formatted) and application-ready JSON files (MTGJSON formatted) with information about spoiled cards from upcoming sets. + +## Output [![Build Status](https://github.com/Cockatrice/Magic-Spoiler/workflows/Deploy/badge.svg?branch=master)](https://github.com/Cockatrice/Magic-Spoiler/actions?query=workflow%3ADeploy+event%3Aworkflow_dispatch+event%3Aschedule+branch%3Amaster) ## +>**Enable "Download Spoilers Automatically" in `Cockatrice → Settings → Card Sources → Spoilers` to get updates automatically pushed to your client!**
+You can also [add the desired .xml file(s) to your customsets folder manually](https://github.com/Cockatrice/Cockatrice/wiki/Custom-Cards-&-Sets#to-add-custom-sets-follow-these-steps) to make Cockatrice use it. -## Output [![Build Status](https://travis-ci.org/Cockatrice/Magic-Spoiler.svg?branch=master)](https://travis-ci.org/Cockatrice/Magic-Spoiler) ## Just looking for XML or JSON files? [They are in our `files` branch!](https://github.com/Cockatrice/Magic-Spoiler/tree/files) -When run by Travis, the script automatically updates the files and uploads new versions there. ([History of changes](https://github.com/Cockatrice/Magic-Spoiler/commits/files))
-Travis CI is run daily on a cron job basis. +When run by our CI, the script automatically updates the files and uploads new versions to this branch. ([History of changes](https://github.com/Cockatrice/Magic-Spoiler/commits/files))
+GitHub Actions are scheduled to autoamtically run on a daily basis. -## Errors ## -Noticed an error in the card data? Check out our [Contributing file](https://github.com/Cockatrice/Magic-Spoiler/blob/master/.github/CONTRIBUTING.md) for information on how to help! +## Contributing ## +Noticed an error in the card data? Check out our [Contributing file](https://github.com/Cockatrice/Magic-Spoiler/blob/master/.github/CONTRIBUTING.md) for information on how to help fixing it! + +We do happily accept PR's that improve our script as well! ## Running ## @@ -33,13 +38,11 @@ pip install -r requirements.txt $> python -m magic_spoiler ``` -Outputs the following files to `out/` directory:
-`spoiler.xml`, `spoiler.json`
-`{SET_CODE}.xml`, `{SET_CODE}.json` -> **spoiler** → files contain all currently available spoilers from different sets
-> **{SETCODE}** → files contain just the spoiler available for this single set
+### Output ### -
+All XML and JSON spoiler files are written to the `out/` directory: -**Enable "Download Spoilers Automatically" in `Cockatrice → Settings → Card Sources → Spoilers` to get updates automatically pushed to your client!**
-You can also [add the desired .xml file(s) to your customsets folder manually](https://github.com/Cockatrice/Cockatrice/wiki/Custom-Cards-&-Sets#to-add-custom-sets-follow-these-steps) to make Cockatrice use it. +| File Name | Content | +|:--|:--| +| `spoiler.xml`, `spoiler.json` | files contain **all** currently available spoilers from different **sets** | +| `{SET_CODE}.xml`, `{SET_CODE}.json` | files contain just the spoiler available for this **single set** | diff --git a/magic_spoiler/__main__.py b/magic_spoiler/__main__.py index d219d375..2704a2b8 100644 --- a/magic_spoiler/__main__.py +++ b/magic_spoiler/__main__.py @@ -7,7 +7,6 @@ import hashlib import json import pathlib import shutil -import sys import time from typing import IO, Any, Dict, List, Tuple, Union @@ -18,9 +17,15 @@ from lxml import etree SCRYFALL_SET_URL: str = "https://api.scryfall.com/sets/{}" SESSION: contextvars.ContextVar = contextvars.ContextVar("SESSION_SCRYFALL") SPOILER_SETS: contextvars.ContextVar = contextvars.ContextVar("SPOILER_SETS") +SPOILER_MARK = "~" OUTPUT_DIR = pathlib.Path("out") OUTPUT_TMP_DIR = OUTPUT_DIR.joinpath("tmp") +XML_ESCAPE_TRANSLATE_MAP = str.maketrans( + {"&": "&", '"': """, "<": "<", ">": ">", + # remove any control characters outright + **{chr(i): "" for i in range(ord(" "))}} +) def __get_session() -> Union[requests.Session, Any]: @@ -203,8 +208,7 @@ def open_header(card_xml_file: IO[Any]) -> None: :param card_xml_file: Card file path """ card_xml_file.write( - "\n" - + "\n" + + "\n" + " \n" @@ -251,11 +255,15 @@ def close_xml_file(card_xml_file: IO[Any]) -> None: card_xml_file.write("\n\n") card_xml_file.close() - # Make the files pretty + # Make the files pretty and add xml declaration parser = etree.XMLParser(remove_blank_text=True) root = etree.parse(card_xml_file.name, parser).getroot() with pathlib.Path(card_xml_file.name).open("wb") as f: - f.write(etree.tostring(root, pretty_print=True)) + f.write(etree.tostring(root, encoding="UTF-8", xml_declaration=True, pretty_print=True)) + + +def xml_escape(text): + return text.translate(XML_ESCAPE_TRANSLATE_MAP) def write_cards( @@ -288,6 +296,11 @@ def write_cards( else: pow_tough = "" + if "loyalty" in card.keys() and card["loyalty"]: + loyalty = str(card["loyalty"]) + else: + loyalty = "" + if "text" in card.keys(): text = card["text"] else: @@ -336,8 +349,28 @@ def write_cards( if card["layout"] == "split" or card["layout"] == "aftermath": continue + set_name, mana_cost, card_cmc, card_type, pow_tough, table_row, text, loyalty = map( + xml_escape, + [set_name, mana_cost, card_cmc, card_type, pow_tough, table_row, text, loyalty], + ) card_xml_file.write("\n") card_xml_file.write("" + set_name + "\n") + card_xml_file.write("" + text + "\n") + card_xml_file.write("\n") + if "colors" in card.keys(): + for color in card["colors"]: + card_xml_file.write("" + str(color) + "\n") + + card_xml_file.write("" + card_type + "\n") + card_xml_file.write("" + card_cmc + "\n") + card_xml_file.write("" + mana_cost + "\n") + if pow_tough: + card_xml_file.write("" + pow_tough + "\n") + + if loyalty: + card_xml_file.write("" + loyalty + "\n") + + card_xml_file.write("\n") card_xml_file.write( '\n" ) - card_xml_file.write("" + mana_cost + "\n") - card_xml_file.write("" + card_cmc + "\n") - - if "colors" in card.keys(): - for color in card["colors"]: - card_xml_file.write("" + str(color) + "\n") - if set_name + " enters the battlefield tapped" in text: card_xml_file.write("1\n") - card_xml_file.write("" + card_type + "\n") - - if pow_tough: - card_xml_file.write("" + pow_tough + "\n") - - if "loyalty" in card.keys(): - card_xml_file.write("" + str(card["loyalty"]) + "\n") card_xml_file.write("" + table_row + "\n") - card_xml_file.write("" + text + "\n") card_xml_file.write("\n") @@ -377,19 +395,19 @@ def write_spoilers_xml(trice_dicts: Dict[str, List[Dict[str, Any]]]) -> bool: """ output_file_name = "spoiler.xml" - pathlib.Path("out").mkdir(exist_ok=True) - card_xml_file = OUTPUT_TMP_DIR.joinpath(output_file_name).open("w") + pathlib.Path("out").mkdir(parents=True, exist_ok=True) + card_xml_file = OUTPUT_TMP_DIR.joinpath(output_file_name).open("w", encoding="utf-8") # Fill in set headers open_header(card_xml_file) for value in SPOILER_SETS.get(): - fill_header_sets(card_xml_file, value) + fill_header_sets(card_xml_file, {key: (value_ + SPOILER_MARK if key == "code" else value_) for key, value_ in value.items()}) close_header(card_xml_file) # Write in all the cards for value in SPOILER_SETS.get(): try: - write_cards(card_xml_file, trice_dicts[value["code"]], value["code"]) + write_cards(card_xml_file, trice_dicts[value["code"]], value["code"] + SPOILER_MARK) except KeyError: print("Skipping " + value["code"]) @@ -417,8 +435,8 @@ def write_spoilers_json(trice_dicts: Dict[str, List[Dict[str, Any]]]) -> bool: output_file_path = OUTPUT_TMP_DIR.joinpath("spoiler.json") - OUTPUT_TMP_DIR.mkdir(exist_ok=True) - with output_file_path.open("w") as f: + OUTPUT_TMP_DIR.mkdir(parents=True, exist_ok=True) + with output_file_path.open("w", encoding="utf-8") as f: json.dump(trice_dicts, f, sort_keys=True, indent=4) # If content didn't change, discard newest creation @@ -445,8 +463,8 @@ def compare_json_content(f1: str, f2: str) -> bool: file2 = pathlib.Path(f2) if file1.is_file() and file2.is_file(): - f1_hash = hashlib.sha512(file1.open("r").read().encode()).hexdigest() - f2_hash = hashlib.sha512(file2.open("r").read().encode()).hexdigest() + f1_hash = hashlib.sha512(file1.open("rb").read()).hexdigest() + f2_hash = hashlib.sha512(file2.open("rb").read()).hexdigest() return f1_hash == f2_hash @@ -490,8 +508,10 @@ def write_set_xml(trice_dict: List[Dict[str, Any]], set_obj: Dict[str, str]) -> if not trice_dict: return False - OUTPUT_TMP_DIR.mkdir(exist_ok=True) - card_xml_file = OUTPUT_TMP_DIR.joinpath("{}.xml".format(set_obj["code"])).open("w") + OUTPUT_TMP_DIR.mkdir(parents=True, exist_ok=True) + set_code = set_obj["code"] + file_path = OUTPUT_TMP_DIR.joinpath(f"{set_code}.xml") + card_xml_file = file_path.open("w", encoding="utf-8") open_header(card_xml_file) fill_header_sets(card_xml_file, set_obj) @@ -527,8 +547,8 @@ def write_set_json(trice_dict: List[Dict[str, Any]], set_obj: Dict[str, str]) -> output_file_path = OUTPUT_TMP_DIR.joinpath("{}.json".format(set_obj["code"])) - OUTPUT_TMP_DIR.mkdir(exist_ok=True) - with output_file_path.open("w") as f: + OUTPUT_TMP_DIR.mkdir(parents=True, exist_ok=True) + with output_file_path.open("w", encoding="utf-8") as f: json.dump(trice_dict, f, sort_keys=True, indent=4) # If content didn't change, discard newest creation @@ -593,10 +613,11 @@ def delete_old_files() -> bool: if OUTPUT_TMP_DIR.is_dir(): shutil.rmtree(OUTPUT_TMP_DIR) + enabled_path = OUTPUT_DIR.joinpath("SpoilerSeasonEnabled") if not SPOILER_SETS.get(): - OUTPUT_DIR.joinpath("SpoilerSeasonEnabled").unlink() + enabled_path.unlink(missing_ok=True) else: - OUTPUT_DIR.joinpath("SpoilerSeasonEnabled").open("w").write(" ") + enabled_path.open("w", encoding="utf-8").write(" ") return deleted @@ -632,9 +653,12 @@ def main() -> None: # Cleanup outdated stuff that's not necessary changed |= delete_old_files() - # Set nonzero exit code if files haven't changed - if not changed: - sys.exit(1) + # Enable deployment on changes (used in CI) + if changed: + print("::set-output name=deploy::true") + else: + print("::set-output name=deploy::false") + print("::notice title=No updates available::No new spoiler cards found for deployment") if __name__ == "__main__":