Merge branch 'master' into tooomm-patch-2

2026-08-20 08:24:59 -05:00 · 2022-02-06 22:59:28 +01:00
parent 63fe156674 bd910e37de
commit 3f0c4878c6
5 changed files with 118 additions and 91 deletions
--- a/.ci/deploy.sh
+++ b/.ci/deploy.sh
@@ -1,33 +0,0 @@
-#!/bin/bash
-set -e # Exit with nonzero exit code if anything fails
-
-function doCompile {
-  echo "Running script..."
-  python3 -m magic_spoiler
-}
-
-# Pull requests and commits to other branches shouldn't try to deploy, just build to verify
-if [[ ! -d $OUTPUT_PATH ]]; then
-  mkdir "$OUTPUT_PATH"
-  echo "Skipping deploy; just doing a build."
-  # Run our compile script and let user know in logs
-  doCompile
-  exit 0
-fi
-
-# Run our compile script and exit gracefully if there are no updates
-if ! doCompile; then
-  echo "::warning::No changes!"
-  exit 0
-fi
-
-cd "$OUTPUT_PATH"
-git config user.name github-actions
-git config user.email github-actions@github.com
-
-# We don't want the AllSets... waste of space
-git add -A .
-git commit -m "Deploy: ${GITHUB_SHA}"
-
-# push using built-in token
-git push
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -1,6 +1,7 @@
 name: Deploy

 on:
+  workflow_dispatch:
  push:
    branches:
      - master
@@ -12,14 +13,19 @@ on:
    paths-ignore:
      - '**.md'
  schedule:
-    # every 6 hours = 4 times a day
-    - cron: '0 */6 * * *'
+    # Every 8 hours = 3 times a day
+    - cron: '0 */8 * * *'

 jobs:
  deploy:
+    # Do not run the scheduled workflow on forks
+    if: github.event_name != 'schedule' || github.repository_owner == 'Cockatrice'
+
+    name: Check for new spoiler
    runs-on: ubuntu-latest

    env:
+      DEPLOY: ${{github.ref == 'refs/heads/master'}}
      OUTPUT_PATH: out

    steps:
@@ -27,7 +33,8 @@ jobs:
        uses: actions/checkout@v2

      - name: Checkout output branch
-        if: github.ref == 'refs/heads/master'
+        # Run only when triggered from master
+        if: env.DEPLOY == 'true'
        uses: actions/checkout@v2
        with:
          ref: files
@@ -42,6 +49,30 @@ jobs:
          python3 -m pip install --upgrade pip setuptools
          python3 -m pip install --requirement requirements.txt

-      - name: Deploy
+      - name: Run script
+        id: run
        shell: bash
-        run: ./.ci/deploy.sh
+        run: python3 -m magic_spoiler
+
+      - name: Upload artifacts
+        # Run only when triggered from a PR
+        if: github.event_name == 'pull_request'
+        uses: actions/upload-artifact@v2
+        with:
+          name: spoiler-output
+          path: ${{github.workspace}}/${{env.OUTPUT_PATH}}
+          if-no-files-found: error
+
+      - name: Deploy changes
+        # Run only when triggered from master and changes are available
+        if: env.DEPLOY == 'true' && steps.run.outputs.deploy == 'true'
+        shell: bash
+        working-directory: ${{env.OUTPUT_PATH}}
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          git add -A .
+          git commit -m "Deploy: $GITHUB_SHA"
+          git push
+          deploy_commit=`git rev-parse HEAD`
+          echo "::notice title=New data uploaded::See deployment: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$deploy_commit"
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,6 @@
 # Project specific
 out/
 AllSets.pre.json
-deploy_key.enc

 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -86,6 +85,9 @@ celerybeat-schedule
 # virtualenv
 venv/
 ENV/
+bin/
+include/
+pyvenv.cfg

 # Spyder project settings
 .spyderproject
@@ -100,4 +102,4 @@ ENV/
 .DS_Store

 *.sqlite
-.*_cache
+.*_cache
--- a/README.md
+++ b/README.md
@@ -6,16 +6,21 @@

 # Magic-Spoiler [![Discord](https://img.shields.io/discord/314987288398659595?label=Discord&logo=discord&logoColor=white)](https://discord.gg/3Z9yzmA) [![Gitter Chat](https://img.shields.io/gitter/room/Cockatrice/Magic-Spoiler)](https://gitter.im/Cockatrice/Magic-Spoiler) #

-Magic-Spoiler is a Python script to scrape <i>[Scryfall](https://scryfall.com)</i> to compile XML files (Cockatrice formatted) and application-ready json files (mtgjson formatted) with information about spoiled cards from upcoming sets.
+Magic-Spoiler is a Python script to query the <i>[Scryfall](https://scryfall.com)</i> API to compile XML files (Cockatrice formatted) and application-ready JSON files (MTGJSON formatted) with information about spoiled cards from upcoming sets.
+
+## Output [![Build Status](https://github.com/Cockatrice/Magic-Spoiler/workflows/Deploy/badge.svg?branch=master)](https://github.com/Cockatrice/Magic-Spoiler/actions?query=workflow%3ADeploy+event%3Aworkflow_dispatch+event%3Aschedule+branch%3Amaster) ##
+>**Enable "Download Spoilers Automatically" in `Cockatrice → Settings → Card Sources → Spoilers` to get updates automatically pushed to your client!**<br>
+You can also [add the desired <b>.xml</b> file(s) to your <i>customsets</i> folder manually](https://github.com/Cockatrice/Cockatrice/wiki/Custom-Cards-&-Sets#to-add-custom-sets-follow-these-steps) to make Cockatrice use it.

-## Output [![Build Status](https://travis-ci.org/Cockatrice/Magic-Spoiler.svg?branch=master)](https://travis-ci.org/Cockatrice/Magic-Spoiler) ##
 Just looking for XML or JSON files?  [They are in our `files` branch!](https://github.com/Cockatrice/Magic-Spoiler/tree/files) 

-When run by Travis, the script automatically updates the files and uploads new versions there. ([History of changes](https://github.com/Cockatrice/Magic-Spoiler/commits/files))<br>
-Travis CI is run daily on a cron job basis.
+When run by our CI, the script automatically updates the files and uploads new versions to this branch. ([History of changes](https://github.com/Cockatrice/Magic-Spoiler/commits/files))<br>
+GitHub Actions are scheduled to autoamtically run on a daily basis.

-## Errors ##
-Noticed an error in the card data?  Check out our [Contributing file](https://github.com/Cockatrice/Magic-Spoiler/blob/master/.github/CONTRIBUTING.md) for information on how to help!
+## Contributing ##
+Noticed an error in the card data? Check out our [Contributing file](https://github.com/Cockatrice/Magic-Spoiler/blob/master/.github/CONTRIBUTING.md) for information on how to help fixing it!
+
+We do happily accept PR's that improve our script as well!

 ## Running ##

@@ -33,13 +38,11 @@ pip install -r requirements.txt
 $> python -m magic_spoiler
 ```

-Outputs the following files to `out/` directory:<br>
-`spoiler.xml`, `spoiler.json`<br>
-`{SET_CODE}.xml`, `{SET_CODE}.json`
-> **spoiler** → files contain all currently available spoilers from different sets<br>
-> **{SETCODE}** → files contain just the spoiler available for this single set<br>
+### Output ###

-<br>
+All XML and JSON spoiler files are written to the `out/` directory:

-**Enable "Download Spoilers Automatically" in `Cockatrice → Settings → Card Sources → Spoilers` to get updates automatically pushed to your client!**<br>
-You can also [add the desired <b>.xml</b> file(s) to your <i>customsets</i> folder manually](https://github.com/Cockatrice/Cockatrice/wiki/Custom-Cards-&-Sets#to-add-custom-sets-follow-these-steps) to make Cockatrice use it.
+| File Name | Content |
+|:--|:--|
+| `spoiler.xml`, `spoiler.json` | files contain **all** currently available spoilers from different **sets** |
+| `{SET_CODE}.xml`, `{SET_CODE}.json` | files contain just the spoiler available for this **single set** |
--- a/magic_spoiler/main.py
+++ b/magic_spoiler/main.py
@@ -7,7 +7,6 @@ import hashlib
 import json
 import pathlib
 import shutil
-import sys
 import time
 from typing import IO, Any, Dict, List, Tuple, Union

@@ -18,9 +17,15 @@ from lxml import etree
 SCRYFALL_SET_URL: str = "https://api.scryfall.com/sets/{}"
 SESSION: contextvars.ContextVar = contextvars.ContextVar("SESSION_SCRYFALL")
 SPOILER_SETS: contextvars.ContextVar = contextvars.ContextVar("SPOILER_SETS")
+SPOILER_MARK = "~"

 OUTPUT_DIR = pathlib.Path("out")
 OUTPUT_TMP_DIR = OUTPUT_DIR.joinpath("tmp")
+XML_ESCAPE_TRANSLATE_MAP = str.maketrans(
+    {"&": "&amp;", '"': "&quot;", "<": "&lt;", ">": "&gt;",
+    # remove any control characters outright
+    **{chr(i): "" for i in range(ord(" "))}}
+)


 def __get_session() -> Union[requests.Session, Any]:
@@ -203,8 +208,7 @@ def open_header(card_xml_file: IO[Any]) -> None:
    :param card_xml_file: Card file path
    """
    card_xml_file.write(
-        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
-        + "<cockatrice_carddatabase version=\"4\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"https://raw.githubusercontent.com/Cockatrice/Cockatrice/master/doc/carddatabase_v4/cards.xsd\">\n"
+        + "<cockatrice_carddatabase version='4' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xsi:schemaLocation='https://raw.githubusercontent.com/Cockatrice/Cockatrice/master/doc/carddatabase_v4/cards.xsd'>\n"
        + "  <!--\n"
        + "  THIS FILE IS AUTOMATICALLY GENERATED & ALL EDITS WILL BE OVERRIDDEN.\n"
        + "  -->\n"
@@ -251,11 +255,15 @@ def close_xml_file(card_xml_file: IO[Any]) -> None:
    card_xml_file.write("</cards>\n</cockatrice_carddatabase>\n")
    card_xml_file.close()

-    # Make the files pretty
+    # Make the files pretty and add xml declaration
    parser = etree.XMLParser(remove_blank_text=True)
    root = etree.parse(card_xml_file.name, parser).getroot()
    with pathlib.Path(card_xml_file.name).open("wb") as f:
-        f.write(etree.tostring(root, pretty_print=True))
+        f.write(etree.tostring(root, encoding="UTF-8", xml_declaration=True, pretty_print=True))
+
+
+def xml_escape(text):
+    return text.translate(XML_ESCAPE_TRANSLATE_MAP)


 def write_cards(
@@ -288,6 +296,11 @@ def write_cards(
        else:
            pow_tough = ""

+        if "loyalty" in card.keys() and card["loyalty"]:
+            loyalty = str(card["loyalty"])
+        else:
+            loyalty = ""
+
        if "text" in card.keys():
            text = card["text"]
        else:
@@ -336,8 +349,28 @@ def write_cards(
                    if card["layout"] == "split" or card["layout"] == "aftermath":
                        continue

+        set_name, mana_cost, card_cmc, card_type, pow_tough, table_row, text, loyalty = map(
+            xml_escape,
+            [set_name, mana_cost, card_cmc, card_type, pow_tough, table_row, text, loyalty],
+        )
        card_xml_file.write("<card>\n")
        card_xml_file.write("<name>" + set_name + "</name>\n")
+        card_xml_file.write("<text>" + text + "</text>\n")
+        card_xml_file.write("<prop>\n")
+        if "colors" in card.keys():
+            for color in card["colors"]:
+                card_xml_file.write("<color>" + str(color) + "</color>\n")
+
+        card_xml_file.write("<type>" + card_type + "</type>\n")
+        card_xml_file.write("<cmc>" + card_cmc + "</cmc>\n")
+        card_xml_file.write("<manacost>" + mana_cost + "</manacost>\n")
+        if pow_tough:
+            card_xml_file.write("<pt>" + pow_tough + "</pt>\n")
+
+        if loyalty:
+            card_xml_file.write("<loyalty>" + loyalty + "</loyalty>\n")
+
+        card_xml_file.write("</prop>\n")
        card_xml_file.write(
            '<set rarity="'
            + str(card["rarity"])
@@ -347,25 +380,10 @@ def write_cards(
            + str(set_code)
            + "</set>\n"
        )
-        card_xml_file.write("<manacost>" + mana_cost + "</manacost>\n")
-        card_xml_file.write("<cmc>" + card_cmc + "</cmc>\n")
-
-        if "colors" in card.keys():
-            for color in card["colors"]:
-                card_xml_file.write("<color>" + str(color) + "</color>\n")
-
        if set_name + " enters the battlefield tapped" in text:
            card_xml_file.write("<cipt>1</cipt>\n")

-        card_xml_file.write("<type>" + card_type + "</type>\n")
-
-        if pow_tough:
-            card_xml_file.write("<pt>" + pow_tough + "</pt>\n")
-
-        if "loyalty" in card.keys():
-            card_xml_file.write("<loyalty>" + str(card["loyalty"]) + "</loyalty>\n")
        card_xml_file.write("<tablerow>" + table_row + "</tablerow>\n")
-        card_xml_file.write("<text>" + text + "</text>\n")
        card_xml_file.write("</card>\n")


@@ -377,19 +395,19 @@ def write_spoilers_xml(trice_dicts: Dict[str, List[Dict[str, Any]]]) -> bool:
    """
    output_file_name = "spoiler.xml"

-    pathlib.Path("out").mkdir(exist_ok=True)
-    card_xml_file = OUTPUT_TMP_DIR.joinpath(output_file_name).open("w")
+    pathlib.Path("out").mkdir(parents=True, exist_ok=True)
+    card_xml_file = OUTPUT_TMP_DIR.joinpath(output_file_name).open("w", encoding="utf-8")

    # Fill in set headers
    open_header(card_xml_file)
    for value in SPOILER_SETS.get():
-        fill_header_sets(card_xml_file, value)
+        fill_header_sets(card_xml_file, {key: (value_ + SPOILER_MARK if key == "code" else value_) for key, value_ in value.items()})
    close_header(card_xml_file)

    # Write in all the cards
    for value in SPOILER_SETS.get():
        try:
-            write_cards(card_xml_file, trice_dicts[value["code"]], value["code"])
+            write_cards(card_xml_file, trice_dicts[value["code"]], value["code"] + SPOILER_MARK)
        except KeyError:
            print("Skipping " + value["code"])

@@ -417,8 +435,8 @@ def write_spoilers_json(trice_dicts: Dict[str, List[Dict[str, Any]]]) -> bool:

    output_file_path = OUTPUT_TMP_DIR.joinpath("spoiler.json")

-    OUTPUT_TMP_DIR.mkdir(exist_ok=True)
-    with output_file_path.open("w") as f:
+    OUTPUT_TMP_DIR.mkdir(parents=True, exist_ok=True)
+    with output_file_path.open("w", encoding="utf-8") as f:
        json.dump(trice_dicts, f, sort_keys=True, indent=4)

    # If content didn't change, discard newest creation
@@ -445,8 +463,8 @@ def compare_json_content(f1: str, f2: str) -> bool:
    file2 = pathlib.Path(f2)

    if file1.is_file() and file2.is_file():
-        f1_hash = hashlib.sha512(file1.open("r").read().encode()).hexdigest()
-        f2_hash = hashlib.sha512(file2.open("r").read().encode()).hexdigest()
+        f1_hash = hashlib.sha512(file1.open("rb").read()).hexdigest()
+        f2_hash = hashlib.sha512(file2.open("rb").read()).hexdigest()

        return f1_hash == f2_hash

@@ -490,8 +508,10 @@ def write_set_xml(trice_dict: List[Dict[str, Any]], set_obj: Dict[str, str]) ->
    if not trice_dict:
        return False

-    OUTPUT_TMP_DIR.mkdir(exist_ok=True)
-    card_xml_file = OUTPUT_TMP_DIR.joinpath("{}.xml".format(set_obj["code"])).open("w")
+    OUTPUT_TMP_DIR.mkdir(parents=True, exist_ok=True)
+    set_code = set_obj["code"]
+    file_path = OUTPUT_TMP_DIR.joinpath(f"{set_code}.xml")
+    card_xml_file = file_path.open("w", encoding="utf-8")

    open_header(card_xml_file)
    fill_header_sets(card_xml_file, set_obj)
@@ -527,8 +547,8 @@ def write_set_json(trice_dict: List[Dict[str, Any]], set_obj: Dict[str, str]) ->

    output_file_path = OUTPUT_TMP_DIR.joinpath("{}.json".format(set_obj["code"]))

-    OUTPUT_TMP_DIR.mkdir(exist_ok=True)
-    with output_file_path.open("w") as f:
+    OUTPUT_TMP_DIR.mkdir(parents=True, exist_ok=True)
+    with output_file_path.open("w", encoding="utf-8") as f:
        json.dump(trice_dict, f, sort_keys=True, indent=4)

    # If content didn't change, discard newest creation
@@ -593,10 +613,11 @@ def delete_old_files() -> bool:
    if OUTPUT_TMP_DIR.is_dir():
        shutil.rmtree(OUTPUT_TMP_DIR)

+    enabled_path = OUTPUT_DIR.joinpath("SpoilerSeasonEnabled")
    if not SPOILER_SETS.get():
-        OUTPUT_DIR.joinpath("SpoilerSeasonEnabled").unlink()
+        enabled_path.unlink(missing_ok=True)
    else:
-        OUTPUT_DIR.joinpath("SpoilerSeasonEnabled").open("w").write(" ")
+        enabled_path.open("w", encoding="utf-8").write(" ")

    return deleted

@@ -632,9 +653,12 @@ def main() -> None:
    # Cleanup outdated stuff that's not necessary
    changed |= delete_old_files()

-    # Set nonzero exit code if files haven't changed
-    if not changed:
-        sys.exit(1)
+    # Enable deployment on changes (used in CI)
+    if changed:
+        print("::set-output name=deploy::true")
+    else:
+        print("::set-output name=deploy::false")
+        print("::notice title=No updates available::No new spoiler cards found for deployment")


 if __name__ == "__main__":