MTGJSON Compressed rules json generator

gen_rules.exs

#! /usr/bin/env elixir

## Requirements
# Elixir     >=1.12 (we use Mix.install/2)
# Erlang/OTP >=22
# p7zip      >=17.04

## About this file
# Everthing above the block comment below is configuration using elixir
# expression syntax. Everything below the block comment is the code that
# generates the json file from the rules.
#
# Comments start with a #
#
# ALWAYS use "" for strings. '' means charlist in elixir, and it is usually
# incompatible with functions expecting strings.
#
# See the following pages for more detailed info
# https://hexdocs.pm/elixir/naming-conventions.html
# https://hexdocs.pm/elixir/operators.html#general-operators
# https://hexdocs.pm/elixir/syntax-reference.html#data-types

# Rules version
# This can either be ':current' or a date formatted like "YYYYMMDD", corresponding with
# the release date of the rules (NOT the effective date)
rule_vsn = :current

# Where to save the rules when we download them and convert them
# Paths not starting with '/' are assumed to be relative
prefix = ""

# What to call the generated JSON file
json_filename_no_ext = "rules"

# Which JSON backend to use
# If you have issues around a missing C compiler, change
# :jiffy
# to
# :jason
# Jiffy is _much_ faster than Jason, but it requires extra tooling to build
json_app = :jiffy

# ---------------------------------------------- #
# You shouldnt need to change anthing below here #
# ---------------------------------------------- #

require Logger

json_spec =
  case json_app do
    :jiffy -> [jiffy: "~> 1.0"]
    :jason -> [jason: "~> 1.2"]
  end

# note that any change to this expression can recompile all deps
deps =
  [
    {:lib_judge, "~> 0.4.1"}
  ] ++ json_spec

Mix.install(deps)

defmodule Json do
  @moduledoc """
  Proxy module to allow for multiple json backends
  """
  case json_app do
    :jiffy ->
      defdelegate decode(struct, opts \\ []), to: :jiffy
      defdelegate encode(struct, opts \\ []), to: :jiffy

    :jason ->
      defdelegate decode(struct, opts \\ []), to: Jason
      defdelegate encode(struct, opts \\ []), to: Jason
  end
end

defmodule SevenZ do
  @moduledoc """
  Light CLI wrapper for 7z
  """
  @common_args ["-mx9"]
  @bz2_args ["-tbzip2", "-mmt=on" | @common_args]
  @gz_args ["-tgzip" | @common_args]
  @xz_args ["-txz", "-mmt=on" | @common_args]
  @zip_args ["-tzip", "-mm=Deflate" | @common_args]

  def compress(algo, data, {json, path}) do
    args = args_for(algo) ++ ["a", path, json]
    System.cmd("7z", args, parallelism: true)
  end

  defp args_for(algo) do
    case algo do
      :bz2 -> @bz2_args
      :gz -> @gz_args
      :xz -> @xz_args
      :zip -> @zip_args
    end
  end
end

defmodule MTGJSON.Rule do
  # TODO, pending lib_judge improvements:
  # references: [rule_id]
  @enforce_keys [:id, :type, :body, :examples]
  defstruct [:id, :type, :body, :examples]

  @type t :: %__MODULE__{
          id: String.t(),
          type: LibJudge.Rule.rule_type(),
          body: String.t(),
          examples: [] | [String.t()]
          # references: [String.t()]
        }

  def from_token({:rule, {rule_type, rule_id_struct, body, examples}}) do
    %__MODULE__{
      id: LibJudge.Rule.to_string!(rule_id_struct),
      type: Atom.to_string(rule_type),
      body: body,
      examples: examples
    }
  end

  def to_map(%MTGJSON.Rule{} = struct) do
    case struct.examples do
      [] ->
        %{
          struct.id => %{
            type: struct.type,
            body: struct.body
          }
        }

      e ->
        %{
          struct.id => %{
            type: struct.type,
            body: struct.body,
            examples: e
          }
        }
    end
  end
end

Logger.info("Fetching and tokenizing rules...")

tokens =
  :current
  |> LibJudge.get!(true, prefix)
  |> LibJudge.tokenize()

Logger.info("Reshaping rule data...")

rule_maps =
  tokens
  |> Stream.filter(LibJudge.Filter.token_type(:rule))
  |> Stream.map(&MTGJSON.Rule.from_token/1)
  |> Stream.map(&MTGJSON.Rule.to_map/1)
  |> Enum.reduce(fn x, acc -> Map.merge(x, acc) end)

effective_date =
  tokens
  |> Keyword.get(:effective_date)
  |> Date.to_string()

Logger.info("Constructing JSON...")

json =
  %{
    effective_date: effective_date,
    rules: rule_maps
  }
  |> Json.encode()

Logger.info("Saving JSON...")

json_path = Path.join(prefix, json_filename_no_ext <> ".json")
File.write!(json_path, json)

Logger.info("Creating compressed JSON files...")

methods = [:zip, :xz, :gz, :bz2]
n_methods = length(methods)

me = self()

for method <- methods do
  file_path = Path.join(prefix, json_filename_no_ext <> ".json." <> Atom.to_string(method))

  task =
    Task.async(fn ->
      SevenZ.compress(method, json, {json_path, file_path})
    end)
end
|> Enum.each(fn task -> Task.await(task) end)

Logger.info("Done!")