From ce18105f1f19cb84e34ebd22b418c1bd732b4f69 Mon Sep 17 00:00:00 2001 From: tooomm Date: Tue, 1 Dec 2020 11:20:48 +0100 Subject: [PATCH 01/51] add GH Actions badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6f9ca614..7ab55777 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Magic-Spoiler is a Python script to scrape [Scryfall](https://scryfall.com) to compile XML files (Cockatrice formatted) and application-ready json files (mtgjson formatted) with information about spoiled cards from upcoming sets. -## Output [![Build Status](https://travis-ci.org/Cockatrice/Magic-Spoiler.svg?branch=master)](https://travis-ci.org/Cockatrice/Magic-Spoiler) ## +## Output [![Build Status](https://github.com/Cockatrice/Magic-Spoiler/workflows/Deploy/badge.svg?branch=master&event=schedule)](https://github.com/Cockatrice/Magic-Spoiler/actions?query=workflow%3ADeploy+event%3Aschedule) ## Just looking for XML or JSON files? [They are in our `files` branch!](https://github.com/Cockatrice/Magic-Spoiler/tree/files) When run by Travis, the script automatically updates the files and uploads new versions there. ([History of changes](https://github.com/Cockatrice/Magic-Spoiler/commits/files))
From 9f3d1839e92ec0ade61371fb7de8d797227f78e3 Mon Sep 17 00:00:00 2001 From: tooomm Date: Tue, 1 Dec 2020 11:22:51 +0100 Subject: [PATCH 02/51] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ab55777..58f3859e 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Magic-Spoiler is a Python script to scrape [Scryfall](https://scryfall.com) to compile XML files (Cockatrice formatted) and application-ready json files (mtgjson formatted) with information about spoiled cards from upcoming sets. -## Output [![Build Status](https://github.com/Cockatrice/Magic-Spoiler/workflows/Deploy/badge.svg?branch=master&event=schedule)](https://github.com/Cockatrice/Magic-Spoiler/actions?query=workflow%3ADeploy+event%3Aschedule) ## +## Output [![Build Status](https://github.com/Cockatrice/Magic-Spoiler/workflows/Deploy/badge.svg?branch=master&event=schedule)](https://github.com/Cockatrice/Magic-Spoiler/actions?query=workflow%3ADeploy+event%3Aschedule+branch%3Amaster) ## Just looking for XML or JSON files? [They are in our `files` branch!](https://github.com/Cockatrice/Magic-Spoiler/tree/files) When run by Travis, the script automatically updates the files and uploads new versions there. ([History of changes](https://github.com/Cockatrice/Magic-Spoiler/commits/files))
From 7fb119e52ed067f27d82e92c1e23f8d1dc030a71 Mon Sep 17 00:00:00 2001 From: tooomm Date: Tue, 1 Dec 2020 11:24:40 +0100 Subject: [PATCH 03/51] add ability to run manually from gui --- .github/workflows/deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b1ba14bb..08a3fb6c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -15,6 +15,8 @@ on: # every 6 hours = 4 times a day - cron: '0 */6 * * *' + workflow_dispatch: + jobs: deploy: runs-on: ubuntu-latest From 8cfbdb7d879b2897483245f15de69a10da48ba19 Mon Sep 17 00:00:00 2001 From: tooomm Date: Tue, 1 Dec 2020 11:37:45 +0100 Subject: [PATCH 04/51] add more context to warning --- .ci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/deploy.sh b/.ci/deploy.sh index 58a7aaea..b5edc206 100755 --- a/.ci/deploy.sh +++ b/.ci/deploy.sh @@ -17,7 +17,7 @@ fi # Run our compile script and exit gracefully if there are no updates if ! doCompile; then - echo "::warning::No changes!" + echo "::warning::No updates found... skipping file upload!" exit 0 fi From aec3dd76186d18a13a65dda5b903232c2b743e60 Mon Sep 17 00:00:00 2001 From: tooomm Date: Wed, 2 Dec 2020 20:24:18 +0100 Subject: [PATCH 05/51] remove deploy key See #258 --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f12822ff..275349cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ # Project specific out/ AllSets.pre.json -deploy_key.enc # Byte-compiled / optimized / DLL files __pycache__/ @@ -100,4 +99,4 @@ ENV/ .DS_Store *.sqlite -.*_cache \ No newline at end of file +.*_cache From a4bdf439ecce7496986e3dbf40149a6afe3ea649 Mon Sep 17 00:00:00 2001 From: tooomm Date: Wed, 2 Dec 2020 20:39:05 +0100 Subject: [PATCH 06/51] one more warning --- .ci/deploy.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.ci/deploy.sh b/.ci/deploy.sh index b5edc206..e5ab0f35 100755 --- a/.ci/deploy.sh +++ b/.ci/deploy.sh @@ -9,7 +9,7 @@ function doCompile { # Pull requests and commits to other branches shouldn't try to deploy, just build to verify if [[ ! -d $OUTPUT_PATH ]]; then mkdir "$OUTPUT_PATH" - echo "Skipping deploy; just doing a build." + echo "::warning::Not running from master... doing a build, but skipping deploy!" # Run our compile script and let user know in logs doCompile exit 0 @@ -24,9 +24,7 @@ fi cd "$OUTPUT_PATH" git config user.name github-actions git config user.email github-actions@github.com - -# We don't want the AllSets... waste of space -git add -A . + git commit -m "Deploy: ${GITHUB_SHA}" # push using built-in token From da60525795da5085c01116c0013c758a319380ce Mon Sep 17 00:00:00 2001 From: tooomm Date: Wed, 2 Dec 2020 22:48:07 +0100 Subject: [PATCH 07/51] fix --- .ci/deploy.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/deploy.sh b/.ci/deploy.sh index e5ab0f35..caf32a12 100755 --- a/.ci/deploy.sh +++ b/.ci/deploy.sh @@ -24,7 +24,9 @@ fi cd "$OUTPUT_PATH" git config user.name github-actions git config user.email github-actions@github.com - + +# We don't want the AllSets... waste of space +git add -A . git commit -m "Deploy: ${GITHUB_SHA}" # push using built-in token From 7a138354f4acbaf6fa0e7e3302c65507e86aad85 Mon Sep 17 00:00:00 2001 From: tooomm Date: Thu, 3 Dec 2020 21:11:55 +0100 Subject: [PATCH 08/51] remove warning --- .ci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/deploy.sh b/.ci/deploy.sh index caf32a12..e11aa1ce 100755 --- a/.ci/deploy.sh +++ b/.ci/deploy.sh @@ -9,7 +9,7 @@ function doCompile { # Pull requests and commits to other branches shouldn't try to deploy, just build to verify if [[ ! -d $OUTPUT_PATH ]]; then mkdir "$OUTPUT_PATH" - echo "::warning::Not running from master... doing a build, but skipping deploy!" + echo "Not running from master... doing a build, but skipping deploy!" # Run our compile script and let user know in logs doCompile exit 0 From 7287cb5fbc2f33ab26864896be18b4245933db04 Mon Sep 17 00:00:00 2001 From: tooomm Date: Thu, 3 Dec 2020 21:37:10 +0100 Subject: [PATCH 09/51] format --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 58f3859e..559fafdb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ # Magic-Spoiler [![Discord](https://img.shields.io/discord/314987288398659595?label=Discord&logo=discord&logoColor=white)](https://discord.gg/3Z9yzmA) [![Gitter Chat](https://img.shields.io/gitter/room/Cockatrice/Magic-Spoiler)](https://gitter.im/Cockatrice/Magic-Spoiler) # -Magic-Spoiler is a Python script to scrape [Scryfall](https://scryfall.com) to compile XML files (Cockatrice formatted) and application-ready json files (mtgjson formatted) with information about spoiled cards from upcoming sets. +Magic-Spoiler is a Python script to scrape [Scryfall](https://scryfall.com) to compile XML files (Cockatrice formatted) and application-ready JSON files (MTGJSON formatted) with information about spoiled cards from upcoming sets. ## Output [![Build Status](https://github.com/Cockatrice/Magic-Spoiler/workflows/Deploy/badge.svg?branch=master&event=schedule)](https://github.com/Cockatrice/Magic-Spoiler/actions?query=workflow%3ADeploy+event%3Aschedule+branch%3Amaster) ## Just looking for XML or JSON files? [They are in our `files` branch!](https://github.com/Cockatrice/Magic-Spoiler/tree/files) From db458c19b78c6742c5af7fd8ace64c1abeabb9dd Mon Sep 17 00:00:00 2001 From: tooomm Date: Thu, 3 Dec 2020 21:43:52 +0100 Subject: [PATCH 10/51] last travis pieces --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 559fafdb..750fb2ca 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ Magic-Spoiler is a Python script to scrape [Scryfall](https://scryfall.com) -Travis CI is run daily on a cron job basis. +When run by our CI, the script automatically updates the files and uploads new versions to this branch. ([History of changes](https://github.com/Cockatrice/Magic-Spoiler/commits/files))
+GitHub Actions are scheduled to autoamtically run on a daily basis. ## Errors ## Noticed an error in the card data? Check out our [Contributing file](https://github.com/Cockatrice/Magic-Spoiler/blob/master/.github/CONTRIBUTING.md) for information on how to help! From 37f2d8e19bb43cda31298bd6e9ca7a77355bc6b9 Mon Sep 17 00:00:00 2001 From: ebbit1q Date: Thu, 3 Dec 2020 23:37:38 +0100 Subject: [PATCH 11/51] remove deploy.sh use output instead of exit code --- .ci/deploy.sh | 33 --------------------------------- .github/workflows/deploy.yml | 22 +++++++++++++++++----- magic_spoiler/__main__.py | 9 ++++----- 3 files changed, 21 insertions(+), 43 deletions(-) delete mode 100755 .ci/deploy.sh diff --git a/.ci/deploy.sh b/.ci/deploy.sh deleted file mode 100755 index e11aa1ce..00000000 --- a/.ci/deploy.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -e # Exit with nonzero exit code if anything fails - -function doCompile { - echo "Running script..." - python3 -m magic_spoiler -} - -# Pull requests and commits to other branches shouldn't try to deploy, just build to verify -if [[ ! -d $OUTPUT_PATH ]]; then - mkdir "$OUTPUT_PATH" - echo "Not running from master... doing a build, but skipping deploy!" - # Run our compile script and let user know in logs - doCompile - exit 0 -fi - -# Run our compile script and exit gracefully if there are no updates -if ! doCompile; then - echo "::warning::No updates found... skipping file upload!" - exit 0 -fi - -cd "$OUTPUT_PATH" -git config user.name github-actions -git config user.email github-actions@github.com - -# We don't want the AllSets... waste of space -git add -A . -git commit -m "Deploy: ${GITHUB_SHA}" - -# push using built-in token -git push diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 08a3fb6c..8dc6cb91 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,6 +1,7 @@ name: Deploy on: + workflow_dispatch: push: branches: - master @@ -15,13 +16,12 @@ on: # every 6 hours = 4 times a day - cron: '0 */6 * * *' - workflow_dispatch: - jobs: deploy: runs-on: ubuntu-latest env: + DEPLOY: ${{github.ref == 'refs/heads/master'}} OUTPUT_PATH: out steps: @@ -29,7 +29,7 @@ jobs: uses: actions/checkout@v2 - name: Checkout output branch - if: github.ref == 'refs/heads/master' + if: env.DEPLOY uses: actions/checkout@v2 with: ref: files @@ -44,6 +44,18 @@ jobs: python3 -m pip install --upgrade pip setuptools python3 -m pip install --requirement requirements.txt - - name: Deploy + - name: Run + id: run shell: bash - run: ./.ci/deploy.sh + run: python3 -m magic_spoiler + + - name: Deploy + if: env.DEPLOY && steps.run.outputs.deploy == true + shell: bash + working-directory: ${{env.OUTPUT_PATH}} + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git add -A . + git commit -m "Deploy: $GITHUB_SHA" + git push diff --git a/magic_spoiler/__main__.py b/magic_spoiler/__main__.py index b8aef5ee..2c51b167 100644 --- a/magic_spoiler/__main__.py +++ b/magic_spoiler/__main__.py @@ -7,7 +7,6 @@ import hashlib import json import pathlib import shutil -import sys import time from typing import IO, Any, Dict, List, Tuple, Union @@ -525,7 +524,7 @@ def write_set_json(trice_dict: List[Dict[str, Any]], set_obj: Dict[str, str]) -> output_file_path = OUTPUT_TMP_DIR.joinpath("{}.json".format(set_obj["code"])) - OUTPUT_TMP_DIR.mkdir(exist_ok=True) + OUTPUT_TMP_DIR.mkdir(parents=True, exist_ok=True) with output_file_path.open("w") as f: json.dump(trice_dict, f, sort_keys=True, indent=4) @@ -630,9 +629,9 @@ def main() -> None: # Cleanup outdated stuff that's not necessary changed |= delete_old_files() - # Set nonzero exit code if files haven't changed - if not changed: - sys.exit(1) + # Set output to deploy + if changed: + print("::set-output name=deploy::true") if __name__ == "__main__": From 91a615f77ddbf8fb70bbfe1c7ac40f947d292791 Mon Sep 17 00:00:00 2001 From: ebbit1q Date: Fri, 4 Dec 2020 01:44:55 +0100 Subject: [PATCH 12/51] do not run the scheduled run on forks --- .github/workflows/deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8dc6cb91..59c61410 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -18,6 +18,9 @@ on: jobs: deploy: + # do not run the sheduled run on forks + if: github.event != 'schedule' || github.repository_owner == 'Cockatrice' + runs-on: ubuntu-latest env: From 9481e5ff243071ea89a26bdbf6a4cc165241048b Mon Sep 17 00:00:00 2001 From: ebbit1q Date: Fri, 4 Dec 2020 03:09:55 +0100 Subject: [PATCH 13/51] correctly check output true != 'true' --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 59c61410..a5e49649 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,7 +53,7 @@ jobs: run: python3 -m magic_spoiler - name: Deploy - if: env.DEPLOY && steps.run.outputs.deploy == true + if: env.DEPLOY && steps.run.outputs.deploy != null shell: bash working-directory: ${{env.OUTPUT_PATH}} run: | From 7c791d4f018769c207f4ba7ce2aeae20fe2351c8 Mon Sep 17 00:00:00 2001 From: tooomm Date: Fri, 4 Dec 2020 17:48:59 +0100 Subject: [PATCH 14/51] step names --- .github/workflows/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a5e49649..2cc9edf5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -47,12 +47,12 @@ jobs: python3 -m pip install --upgrade pip setuptools python3 -m pip install --requirement requirements.txt - - name: Run + - name: Run script id: run shell: bash run: python3 -m magic_spoiler - - name: Deploy + - name: Deploy changes if: env.DEPLOY && steps.run.outputs.deploy != null shell: bash working-directory: ${{env.OUTPUT_PATH}} From fd65e743cc31d92aae21f2300fda37473c7eb989 Mon Sep 17 00:00:00 2001 From: ebbit1q Date: Sun, 17 Jan 2021 01:27:31 +0100 Subject: [PATCH 15/51] the suggested changes --- .github/workflows/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2cc9edf5..414f3861 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v2 - name: Checkout output branch - if: env.DEPLOY + if: env.DEPLOY == 'true' uses: actions/checkout@v2 with: ref: files @@ -53,7 +53,7 @@ jobs: run: python3 -m magic_spoiler - name: Deploy changes - if: env.DEPLOY && steps.run.outputs.deploy != null + if: env.DEPLOY == 'true' && steps.run.outputs.deploy != null shell: bash working-directory: ${{env.OUTPUT_PATH}} run: | From a242ebb1d1a4d728c8cd47dc568680ec7fdf7d83 Mon Sep 17 00:00:00 2001 From: ebbit1q Date: Sun, 17 Jan 2021 04:26:18 +0100 Subject: [PATCH 16/51] escape some xml values --- magic_spoiler/__main__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/magic_spoiler/__main__.py b/magic_spoiler/__main__.py index 2c51b167..5f817aba 100644 --- a/magic_spoiler/__main__.py +++ b/magic_spoiler/__main__.py @@ -20,6 +20,9 @@ SPOILER_SETS: contextvars.ContextVar = contextvars.ContextVar("SPOILER_SETS") OUTPUT_DIR = pathlib.Path("out") OUTPUT_TMP_DIR = OUTPUT_DIR.joinpath("tmp") +XML_ESCAPE_TRANSLATE_MAP = str.maketrans( + {"&": "&", '"': """, "<": "<", ">": ">"} +) def __get_session() -> Union[requests.Session, Any]: @@ -255,6 +258,10 @@ def close_xml_file(card_xml_file: IO[Any]) -> None: f.write(etree.tostring(root, pretty_print=True)) +def xml_escape(text): + return text.translate(XML_ESCAPE_TRANSLATE_MAP) + + def write_cards( card_xml_file: Any, trice_dict: List[Dict[str, Any]], set_code: str ) -> None: @@ -333,6 +340,10 @@ def write_cards( if card["layout"] == "split" or card["layout"] == "aftermath": continue + set_name, mana_cost, card_cmc, card_type, pow_tough, table_row, text = map( + xml_escape, + [set_name, mana_cost, card_cmc, card_type, pow_tough, table_row, text], + ) card_xml_file.write("\n") card_xml_file.write("" + set_name + "\n") card_xml_file.write( From b5d1d938a8b122a0788f87e74c2d70ba8c44708d Mon Sep 17 00:00:00 2001 From: ebbit1q Date: Sun, 17 Jan 2021 04:44:52 +0100 Subject: [PATCH 17/51] add -p to all mkdirs --- magic_spoiler/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/magic_spoiler/__main__.py b/magic_spoiler/__main__.py index 5f817aba..1c0c799a 100644 --- a/magic_spoiler/__main__.py +++ b/magic_spoiler/__main__.py @@ -385,7 +385,7 @@ def write_spoilers_xml(trice_dicts: Dict[str, List[Dict[str, Any]]]) -> bool: """ output_file_name = "spoiler.xml" - pathlib.Path("out").mkdir(exist_ok=True) + pathlib.Path("out").mkdir(parents=True, exist_ok=True) card_xml_file = OUTPUT_TMP_DIR.joinpath(output_file_name).open("w") # Fill in set headers @@ -425,7 +425,7 @@ def write_spoilers_json(trice_dicts: Dict[str, List[Dict[str, Any]]]) -> bool: output_file_path = OUTPUT_TMP_DIR.joinpath("spoiler.json") - OUTPUT_TMP_DIR.mkdir(exist_ok=True) + OUTPUT_TMP_DIR.mkdir(parents=True, exist_ok=True) with output_file_path.open("w") as f: json.dump(trice_dicts, f, sort_keys=True, indent=4) @@ -498,7 +498,7 @@ def write_set_xml(trice_dict: List[Dict[str, Any]], set_obj: Dict[str, str]) -> if not trice_dict: return False - OUTPUT_TMP_DIR.mkdir(exist_ok=True) + OUTPUT_TMP_DIR.mkdir(parents=True, exist_ok=True) card_xml_file = OUTPUT_TMP_DIR.joinpath("{}.xml".format(set_obj["code"])).open("w") open_header(card_xml_file) From 4b25618657c8ecfd569ed449da503d84f4ae3ef5 Mon Sep 17 00:00:00 2001 From: ebbit1q Date: Fri, 30 Apr 2021 00:32:43 +0200 Subject: [PATCH 18/51] fix exception when spoiler season is inactive --- .github/workflows/deploy.yml | 2 +- .gitignore | 2 ++ magic_spoiler/__main__.py | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 414f3861..8b9387d9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,7 +53,7 @@ jobs: run: python3 -m magic_spoiler - name: Deploy changes - if: env.DEPLOY == 'true' && steps.run.outputs.deploy != null + if: env.DEPLOY == 'true' && steps.run.outputs.deploy == 'true' shell: bash working-directory: ${{env.OUTPUT_PATH}} run: | diff --git a/.gitignore b/.gitignore index 275349cb..aacabd7c 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,8 @@ celerybeat-schedule # virtualenv venv/ ENV/ +bin/ +pyvenv.cfg # Spyder project settings .spyderproject diff --git a/magic_spoiler/__main__.py b/magic_spoiler/__main__.py index 1c0c799a..45fc6386 100644 --- a/magic_spoiler/__main__.py +++ b/magic_spoiler/__main__.py @@ -602,7 +602,7 @@ def delete_old_files() -> bool: shutil.rmtree(OUTPUT_TMP_DIR) if not SPOILER_SETS.get(): - OUTPUT_DIR.joinpath("SpoilerSeasonEnabled").unlink() + OUTPUT_DIR.joinpath("SpoilerSeasonEnabled").unlink(missing_ok=True) else: OUTPUT_DIR.joinpath("SpoilerSeasonEnabled").open("w").write(" ") @@ -643,6 +643,8 @@ def main() -> None: # Set output to deploy if changed: print("::set-output name=deploy::true") + else: + print("::set-output name=deploy::false") if __name__ == "__main__": From 103279671d20f74c15dac547dd0f89404a9e7a5f Mon Sep 17 00:00:00 2001 From: ebbit1q Date: Thu, 27 May 2021 23:22:36 +0200 Subject: [PATCH 19/51] add include/ to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index aacabd7c..0fa3782e 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,7 @@ celerybeat-schedule venv/ ENV/ bin/ +include/ pyvenv.cfg # Spyder project settings From 0f4ec17eb6a4853905f9cd84342fe1158c332e59 Mon Sep 17 00:00:00 2001 From: ebbit1q Date: Thu, 27 May 2021 23:23:10 +0200 Subject: [PATCH 20/51] remove control characters from xml --- magic_spoiler/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/magic_spoiler/__main__.py b/magic_spoiler/__main__.py index 45fc6386..b1685b7f 100644 --- a/magic_spoiler/__main__.py +++ b/magic_spoiler/__main__.py @@ -21,7 +21,9 @@ SPOILER_SETS: contextvars.ContextVar = contextvars.ContextVar("SPOILER_SETS") OUTPUT_DIR = pathlib.Path("out") OUTPUT_TMP_DIR = OUTPUT_DIR.joinpath("tmp") XML_ESCAPE_TRANSLATE_MAP = str.maketrans( - {"&": "&", '"': """, "<": "<", ">": ">"} + {"&": "&", '"': """, "<": "<", ">": ">", + # remove any control characters outright + **{chr(i): "" for i in range(ord(" "))}} ) From 31ba735d95296e7754f320478b62d7a5628c6f12 Mon Sep 17 00:00:00 2001 From: ebbit1q Date: Thu, 27 May 2021 23:23:34 +0200 Subject: [PATCH 21/51] add asterisk to spoiler set codes this will make them not use the same location on disk anymore so people will complain less about keeping spoiler card art when they don't clear their image cache people should still clear their image cache this is a workaround, a nicer solution should be implemented in cockatrice sometime --- magic_spoiler/__main__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/magic_spoiler/__main__.py b/magic_spoiler/__main__.py index b1685b7f..296c53f4 100644 --- a/magic_spoiler/__main__.py +++ b/magic_spoiler/__main__.py @@ -17,6 +17,7 @@ from lxml import etree SCRYFALL_SET_URL: str = "https://api.scryfall.com/sets/{}" SESSION: contextvars.ContextVar = contextvars.ContextVar("SESSION_SCRYFALL") SPOILER_SETS: contextvars.ContextVar = contextvars.ContextVar("SPOILER_SETS") +SPOILER_MARK = "*" OUTPUT_DIR = pathlib.Path("out") OUTPUT_TMP_DIR = OUTPUT_DIR.joinpath("tmp") @@ -393,13 +394,13 @@ def write_spoilers_xml(trice_dicts: Dict[str, List[Dict[str, Any]]]) -> bool: # Fill in set headers open_header(card_xml_file) for value in SPOILER_SETS.get(): - fill_header_sets(card_xml_file, value) + fill_header_sets(card_xml_file, {key: (value_ + SPOILER_MARK if key == "code" else value_) for key, value_ in value.items()}) close_header(card_xml_file) # Write in all the cards for value in SPOILER_SETS.get(): try: - write_cards(card_xml_file, trice_dicts[value["code"]], value["code"]) + write_cards(card_xml_file, trice_dicts[value["code"]], value["code"] + SPOILER_MARK) except KeyError: print("Skipping " + value["code"]) From 26e1e22ad0c4141d0a049fa8c30e96d3e1050962 Mon Sep 17 00:00:00 2001 From: ebbit1q Date: Fri, 28 May 2021 00:43:26 +0200 Subject: [PATCH 22/51] use tilde (`~`) as the SPOILER_MARKER instead * is apparently not allowed as a filename on windows --- magic_spoiler/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magic_spoiler/__main__.py b/magic_spoiler/__main__.py index 296c53f4..9c2474a3 100644 --- a/magic_spoiler/__main__.py +++ b/magic_spoiler/__main__.py @@ -17,7 +17,7 @@ from lxml import etree SCRYFALL_SET_URL: str = "https://api.scryfall.com/sets/{}" SESSION: contextvars.ContextVar = contextvars.ContextVar("SESSION_SCRYFALL") SPOILER_SETS: contextvars.ContextVar = contextvars.ContextVar("SPOILER_SETS") -SPOILER_MARK = "*" +SPOILER_MARK = "~" OUTPUT_DIR = pathlib.Path("out") OUTPUT_TMP_DIR = OUTPUT_DIR.joinpath("tmp") From d365ca0375ccc3f34980950f67952017cd033b30 Mon Sep 17 00:00:00 2001 From: ebbit1q Date: Wed, 14 Jul 2021 12:05:00 +0200 Subject: [PATCH 23/51] move settings info to a more visible location (#274) * move settings info to a more visible location * put in quotes Co-authored-by: tooomm --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 750fb2ca..12762415 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ Magic-Spoiler is a Python script to scrape [Scryfall](https://scryfall.com) to compile XML files (Cockatrice formatted) and application-ready JSON files (MTGJSON formatted) with information about spoiled cards from upcoming sets. ## Output [![Build Status](https://github.com/Cockatrice/Magic-Spoiler/workflows/Deploy/badge.svg?branch=master&event=schedule)](https://github.com/Cockatrice/Magic-Spoiler/actions?query=workflow%3ADeploy+event%3Aschedule+branch%3Amaster) ## +>**Enable "Download Spoilers Automatically" in `Cockatrice → Settings → Card Sources → Spoilers` to get updates automatically pushed to your client!**
+You can also [add the desired .xml file(s) to your customsets folder manually](https://github.com/Cockatrice/Cockatrice/wiki/Custom-Cards-&-Sets#to-add-custom-sets-follow-these-steps) to make Cockatrice use it. + Just looking for XML or JSON files? [They are in our `files` branch!](https://github.com/Cockatrice/Magic-Spoiler/tree/files) When run by our CI, the script automatically updates the files and uploads new versions to this branch. ([History of changes](https://github.com/Cockatrice/Magic-Spoiler/commits/files))
@@ -38,8 +41,3 @@ Outputs the following files to `out/` directory:
`{SET_CODE}.xml`, `{SET_CODE}.json` > **spoiler** → files contain all currently available spoilers from different sets
> **{SETCODE}** → files contain just the spoiler available for this single set
- -
- -**Enable "Download Spoilers Automatically" in `Cockatrice → Settings → Card Sources → Spoilers` to get updates automatically pushed to your client!**
-You can also [add the desired .xml file(s) to your customsets folder manually](https://github.com/Cockatrice/Cockatrice/wiki/Custom-Cards-&-Sets#to-add-custom-sets-follow-these-steps) to make Cockatrice use it. From ff54e754c8fb801b4c20d54d9aca3f8eb772bc96 Mon Sep 17 00:00:00 2001 From: tooomm Date: Thu, 15 Jul 2021 20:27:58 +0200 Subject: [PATCH 24/51] re-add xml version and encoding information --- magic_spoiler/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/magic_spoiler/__main__.py b/magic_spoiler/__main__.py index 9c2474a3..e5cd30f8 100644 --- a/magic_spoiler/__main__.py +++ b/magic_spoiler/__main__.py @@ -208,6 +208,7 @@ def open_header(card_xml_file: IO[Any]) -> None: :param card_xml_file: Card file path """ card_xml_file.write( + "\n" "\n" + "