Neural MtG

README.md

Let's play neural MtG!

1. Get the mtg-neural-deck.py script to download some cards.
2. Build a deck with your freshly hallucinated cards.
3. ???
4. Profit!

EDA

Check out the Jupyter notebook eda.ipynb to throw some statistics on your already downloaded cards.

eda.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc49e3d9-bf07-4555-bdc8-6edc5e586cf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import random\n",
    "\n",
    "import pylab as plt\n",
    "\n",
    "from attrdict import AttrDict\n",
    "from glob import glob\n",
    "from IPython.display import JSON\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6efea9ed-1b65-4a5a-b018-ed9096f6bb20",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dir_info = 'cards/info'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90c46118-03bc-424a-90c3-5e1b688dadae",
   "metadata": {},
   "outputs": [],
   "source": [
    "card_types_main = [\n",
    "    'Artifact',\n",
    "    'Creature',\n",
    "    'Enchantment',\n",
    "    'Instant',\n",
    "    'Land',\n",
    "    'Planeswalker',\n",
    "    'Sorcery',\n",
    "]\n",
    "card_types_sub = [\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c964de95-8aef-4385-b3fa-f710e35138b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_card_infos(base_dir):\n",
    "    cards = glob(f\"{base_dir}/*.json\")\n",
    "    card_infos = {}\n",
    "    for c in cards:\n",
    "        cid = Path(c).stem\n",
    "        with open(c) as fn:\n",
    "            card_info = AttrDict(json.load(fn))\n",
    "            card_infos[cid] = card_info\n",
    "    return card_infos\n",
    "\n",
    "card_infos = load_card_infos(base_dir_info)\n",
    "print(f\"Loaded data of a total of {len(card_infos)} cards.\")\n",
    "print()\n",
    "print(\"random card info as an example:\")\n",
    "JSON(card_infos[random.sample(list(card_infos.keys()), 1)[0]], expanded=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ece297f5-5c38-4ce1-bcba-53b8eda9e7df",
   "metadata": {},
   "outputs": [],
   "source": [
    "count_types_main = {\n",
    "    k: sum([1\n",
    "        for v in card_infos.values()\n",
    "        if k in v.type\n",
    "    ])\n",
    "    for k in card_types_main\n",
    "}\n",
    "# JSON(count_types_main)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca01d1bf-6848-41df-96c2-217b11a8284c",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar(range(len(count_types_main)), list(count_types_main.values()), align='center')\n",
    "plt.xticks(range(len(count_types_main)), list(count_types_main.keys()))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8414f96a-222c-4cd4-ad52-21e64c1c28ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "count_mana_costs = {}\n",
    "for c in card_infos.values():\n",
    "    mana_short = c.mana_cost.replace('[', '').replace(']', '')\n",
    "    if not mana_short in count_mana_costs.keys():\n",
    "        count_mana_costs[mana_short] = 1\n",
    "    else:\n",
    "        count_mana_costs[mana_short] += 1\n",
    "# JSON(count_mana_costs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd1c7ab5-8ff4-41c3-8314-c982de30a211",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar(range(len(count_mana_costs)), list(count_mana_costs.values()), align='center')\n",
    "plt.xticks(range(len(count_mana_costs)), list(count_mana_costs.keys()))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2426ebb8-5cd3-4028-9461-9b5fd550ab93",
   "metadata": {},
   "outputs": [],
   "source": [
    "# evergreens = [\n",
    "#     \"Activate\",\n",
    "#     \"Attach\n",
    "#     \"Cast\",\n",
    "#     \"Counter\n",
    "#     \"Create\",\n",
    "#     \"Destroy\",\n",
    "#     \"Discard\",\n",
    "#     \"Exchange\",\n",
    "#     \"Exile\",\n",
    "#     \"Fight\",\n",
    "#     \"Mill\",\n",
    "#     \"Play\n",
    "#     \"Reveal\",\n",
    "#     \"Sacrifice\n",
    "#     \"Scry\",\n",
    "#     \"Search\n",
    "#     \"Shuffle\",\n",
    "#     \"Tap/Untap\",\n",
    "# ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e472316-552b-400a-85ac-e76b0e33e1c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "evergreen_abilities = [\n",
    "    \"Deathtouch\",\n",
    "    \"Defender\",\n",
    "    \"Double strike\",\n",
    "    \"Enchant\",\n",
    "    \"Equip\",\n",
    "    \"First strike\",\n",
    "    \"Flash\",\n",
    "    \"Flying\",\n",
    "    \"Haste\",\n",
    "    \"Hexproof\",\n",
    "    \"Indestructible\",\n",
    "    \"Lifelink\",\n",
    "    \"Menace\",\n",
    "    \"Protection\",\n",
    "    \"Reach\",\n",
    "    \"Trample\",\n",
    "    \"Vigilance\",\n",
    "    \"Ward\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f90a3ff-0968-4546-b5d4-fd25af3fc603",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for c in card_infos.values():\n",
    "    for d in c['description']:\n",
    "        print(d)\n",
    "    break\n",
    "    mana_short = c.mana_cost.replace('[', '').replace(']', '')\n",
    "    if not mana_short in count_mana_costs.keys():\n",
    "        count_mana_costs[mana_short] = 1\n",
    "    else:\n",
    "        count_mana_costs[mana_short] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bf6b14d-46a9-44cb-a9f1-1f3202a24832",
   "metadata": {},
   "outputs": [],
   "source": [
    "count_evergreen_abilities = {k: 0 for k in evergreen_abilities}\n",
    "for c in card_infos.values():\n",
    "    for d in c.description:\n",
    "        for a in evergreen_abilities:\n",
    "            if a in d:\n",
    "                count_evergreen_abilities[a] += 1\n",
    "# JSON(count_evergreen_abilities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34a79aef-85ba-4c4a-a4e6-43bf1e0bfee0",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar(range(len(count_evergreen_abilities)), list(count_evergreen_abilities.values()), align='center')\n",
    "plt.xticks(range(len(count_evergreen_abilities)), list(count_evergreen_abilities.keys()))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e005f373-d25b-43bd-b557-b6210bfb9011",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df6d8ce0-1ad2-4b49-9b22-c4d5826593ed",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eff3eaa3-adfd-47e0-a024-3104d42bc6fe",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39fac646-cdcb-4ed8-9bbf-cfc8c07734a2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf32ba1a-3eb6-46bb-adae-620f14988ff2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29d4f9f4-f22e-4e93-b6cb-fdb400dd7ee7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ad35fde-bb1c-4463-8e7c-1f877ef8e7f3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "232decc6-2b95-4bf8-a48e-e086b1972073",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}

list-latest.sh

#!/usr/bin/env bash
find cards -type f -iname '*.png' | wc -l
echo
while read f; do
    echo -n "${f%%.json}: "
    jq ".name" -r < "cards/info/$f"
done <<< $(ls -htr1 cards/info | tail -5)

mtg-neural-deck.py

#!/usr/bin/env python3
# Automatically download newly generated neural MtG cards for freestyle deck building
# All credits to urzas.ai and respective authors
# Please use with care! Do not create unnecessary load!

import arrow
import io
import json
import pathlib
import PIL
import time
import sys

import urllib.request
import regex as re

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By


card_states = {
    "Penning Words...": 0,
    "Casting Shapes...": 1,
    "Forge New Spell": 2,
}


def wait_for_card(driver, sleep_time=1, max_retries=60):
    old_url = driver.current_url
    
    # Wait for button to show "Forge Another"
    btn_new_card = driver.find_elements(By.ID, "action-button")[0]
    card_info = None
    last_state = None
    retries = 0
    while True:
        if retries > max_retries:
            print(" TIMEOUT :(")
            raise ConnectionAbortedError("Could not generate card in time.")
        txt = btn_new_card.text
        if txt in card_states.keys():
            state = card_states[txt]
        elif txt == "Forge a Spell":
            btn_new_card.click()
            state = None
        else:
            raise ValueError(f"Didn't understand button's label: {txt}")
        # State transition
        if state != last_state:
            # Finished
            if state == 2:
                print(f" {retries * sleep_time}s")
                break 
            else:
                if last_state != None:
                    print(f" {retries * sleep_time}s")
                if state == 1:
                    card_info = get_card_info(driver)
                    print(card_info)
                print(txt, end="")
                sys.stdout.flush()
        elif state != None:
            print(".", end="")
            sys.stdout.flush()
            retries += 1
        last_state = state
        time.sleep(sleep_time)

    # Wait for URL to update via automatic redirect
    print("Generating Image...", end="")
    sys.stdout.flush()
    retries = 0
    while True:
        if retries > max_retries:
            # TODO FIXME NEW WEBSITE DOES NOT REDIRECT ANYMORE?
            print(" TIMEOUT :(")
            raise ConnectionAbortedError("Could not generate card in time.")
            break
        print(".", end="")
        sys.stdout.flush()
        time.sleep(sleep_time)
        try:
            new_url = driver.current_url
        except Exception as e:
            print(e)
            continue
        if new_url != old_url:
            break
        retries += 1
    print(f" {retries * sleep_time}s")
    return card_info


def convert_mana_cost_abbrev(element):
    html = element.get_attribute('innerHTML')
    html_replaced = re.sub(r'<abbr class="[^"]+ card-symbol-([A-Z0-9])" [^>]+>{T}</abbr>', r'[\1]', html)
    return html_replaced.replace('&nbsp;', '')


def get_card_info(driver):
    get_css = lambda css: driver.find_element(By.CSS_SELECTOR, css)
    card_name = get_css('.card-frame > .frame-header > .name').text.strip()
    element_mana_cost = get_css('.card-frame > .frame-header > .mana-cost')
    card_mana_cost = convert_mana_cost_abbrev(element_mana_cost)
    card_type = get_css('.card-frame > .frame-type-line > .type').text
    element_description = get_css('.card-frame > .frame-text-box > .description')
    card_description = convert_mana_cost_abbrev(element_description).split('<br>')
    card_flavour_text = get_css('.card-frame > .frame-text-box > .flavour-text').text
    card_info = {
        'name': card_name.strip(),
        'mana_cost': card_mana_cost.strip(),
        'type': card_type.strip(),
        'description': [s.strip() for s in card_description],
    }
    return card_info


def download_card(driver, img_dir):
    img_card = driver.find_elements(By.CSS_SELECTOR, ".card-side > img")[1]
    img_url = img_card.get_attribute('src')
    img_file = img_url.split('/')[-1].split('?')[0]
    img_path = f"{img_dir}/{img_file}"
    pathlib.Path(img_dir).mkdir(parents=True, exist_ok=True)
    try:
        urllib.request.urlretrieve(img_url, img_path)
        print(f"Saved card's image to {img_path}")
    except TimeoutError:
        print("Timeout while downloading card. :(")
    return img_file


def initialize_driver():
    # Initialize selenium driver
    options = webdriver.chrome.options.Options()
    options.headless = True
    options.add_argument("--window-size=1024,840")
    options.add_argument('--disable-browser-side-navigation') # Remedies hanging when getting current URL?
    driver = webdriver.Chrome(options=options)

    # Visit base URL
    url_start = "https://www.urzas.ai/"
    driver.get(url_start)
    return driver


def save_card_info(card_info, info_file):
    pathlib.Path('/'.join(info_file.split('/')[:-1])).mkdir(parents=True, exist_ok=True)
    json_string = json.dumps(card_info)
    with open(info_file, 'w') as json_file:
        json_file.write(json_string)
    print(f"Saved card's info to {info_file}")


def generate_and_save_card(driver):
    # Click button to generate card
    btn_new_card = driver.find_elements(By.ID, "action-button")[0]
    btn_new_card.click()
    try:
        card_info = wait_for_card(driver)
    except ConnectionAbortedError:
        print("Skipped because of timeout...")
        return False
    img_file = download_card(driver, 'cards/img')
    info_file = img_file.replace('.png', '.json')
    save_card_info(card_info, f'cards/info/{info_file}')
    return True


if __name__ == '__main__':
    print("Initializing selenium driver...")
    n = 5
    driver = initialize_driver()
    last_timestamp = arrow.utcnow()
    for i in range(n):
        print(f"Downloading card {i + 1}/{n}...")
        success = generate_and_save_card(driver)
        success_str = "Successfully" if success else "Failed to"
        timestamp = arrow.utcnow()
        print(f"{success_str} generate and download card within {(timestamp - last_timestamp).seconds}s")
        print()
        last_timestamp = timestamp
    driver.quit()

remove-missing.py

#!/usr/bin/env python3
# Removes files that are either only in cards/info or in cards/img

import os

from glob import glob
from pathlib import Path

cards_img = [Path(f).stem for f in glob('cards/img/*.png')]
cards_info = [Path(f).stem for f in glob('cards/info/*.json')]
cards_img_missing = [
    cards_img[i]
    for i, c in enumerate(cards_img)
    if c not in cards_info
]
cards_info_missing = [
    cards_info[i]
    for i, c in enumerate(cards_info)
    if c not in cards_img
]

if len(cards_img_missing) > 0:
    print("Deleting the following image files, because there is no corresponding info:")
    for c in cards_img_missing:
        print(c)
        os.remove(f"cards/img/{c}.png")

if len(cards_info_missing) > 0:
    print("Deleting the following info files, because there is no corresponding image:")
    for c in cards_info_missing:
        print(c)
        os.remove(f"cards/info/{c}.json")