fetch_all_ethereum_data.ex

defmodule EthData do
  @from ~U[2019-05-01 00:00:00Z]
  @to ~U[2019-08-01 00:00:00Z]
  @slug "ethereum"

  @special [
    "socialGainersLosersStatus",
    "socialVolume",
    {"socialVolume", "mentionsCount"},
    {"ethSpentOverTime", "ethSpent"},
    {"topHoldersPercentOfTotalSupply", "inTopHoldersTotal"},
    {"percentOfTokenSupplyOnExchanges", "percentOnExchanges"}
  ]

  @metrics [
    {"devActivity", "activity"},
    {"githubActivity", "activity"},
    {"historyTwitterData", "followersCount"},
    {"gasUsed", "gasUsed"},
    {"shareOfDeposits", "shareOfDeposits"}
  ]

  @get_metric_metrics [
    "daily_avg_marketcap_usd",
    "daily_avg_price_usd",
    "daily_closing_marketcap_usd",
    "daily_closing_price_usd",
    "daily_high_price_usd",
    "daily_low_price_usd",
    "daily_opening_price_usd",
    "daily_trading_volume_usd",
    "daily_active_addresses",
    "mean_realized_price_usd",
    "mean_realized_price_usd_10y",
    "mean_realized_price_usd_5y",
    "mean_realized_price_usd_3y",
    "mean_realized_price_usd_2y",
    "mean_realized_price_usd_365d",
    "mean_realized_price_usd_180d",
    "mean_realized_price_usd_90d",
    "mean_realized_price_usd_60d",
    "mean_realized_price_usd_30d",
    "mean_realized_price_usd_7d",
    "mean_realized_price_usd_1d",
    "mvrv_usd_long_short_diff",
    "mvrv_usd",
    "mvrv_usd_10y",
    "mvrv_usd_5y",
    "mvrv_usd_3y",
    "mvrv_usd_2y",
    "mvrv_usd_365d",
    "mvrv_usd_180d",
    "mvrv_usd_90d",
    "mvrv_usd_60d",
    "mvrv_usd_30d",
    "mvrv_usd_7d",
    "mvrv_usd_1d",
    "circulation",
    "circulation_10y",
    "circulation_5y",
    "circulation_3y",
    "circulation_2y",
    "circulation_365d",
    "circulation_180d",
    "circulation_90d",
    "circulation_60d",
    "circulation_30d",
    "circulation_7d",
    "circulation_1d",
    "mean_age",
    "realized_value_usd",
    "realized_value_usd_10y",
    "realized_value_usd_5y",
    "realized_value_usd_3y",
    "realized_value_usd_2y",
    "realized_value_usd_365d",
    "realized_value_usd_180d",
    "realized_value_usd_90d",
    "realized_value_usd_60d",
    "realized_value_usd_30d",
    "realized_value_usd_7d",
    "realized_value_usd_1d",
    "velocity",
    "transaction_volume",
    "transaction_volume_5min",
    "exchange_inflow",
    "exchange_outflow",
    "exchange_balance",
    "age_destroyed",
    "age_destroyed_5min",
    "nvt"
  ]
  def run do
    data = get_metrics() ++ get_metrics2()

    columns =
      (["date"] ++ @get_metric_metrics ++ Enum.map(@metrics, &elem(&1, 0))) |> Enum.join(",")

    data_zipped = Enum.zip(data) |> Enum.map(&Tuple.to_list/1)

    csv =
      Enum.reduce(data_zipped, [columns], fn elem, acc ->
        [%{"datetime" => dt} | _] = elem

        {:ok, dt, _} = DateTime.from_iso8601(dt)

        row =
          Enum.reduce(elem, [DateTime.to_date(dt)], fn d, row_tmp ->
            value = Map.delete(d, "datetime") |> Map.values() |> List.first()
            [value | row_tmp]
          end)
          |> Enum.reverse()
          |> Enum.join(",")

        [row | acc]
      end)
      |> Enum.reverse()
      |> Enum.join("\n")

    File.write!(Path.join(__DIR__, "ethereum_data.csv"), csv)
  end

  def get_metrics() do
    Neuron.Config.set(url: "https://api.santiment.net/graphql")
    Enum.map(@get_metric_metrics, &get_metric/1)
  end

  def get_metrics2() do
    Neuron.Config.set(url: "https://api.santiment.net/graphql")
    Enum.map(@metrics, &get_metric2/1)
  end

  defp get_metric(metric) do
    Neuron.Config.set(url: "https://api.santiment.net/graphql")
    Neuron.Config.set(connection_opts: [recv_timeout: 30_000])

    data =
      Neuron.query("""
      {
        getMetric(metric: "#{metric}"){
          timeseriesData(slug: "#{@slug}", from: "#{@from}", to: "#{@to}", interval: "1d") {
            datetime
            value
          }
        }
      }
      """)

    case data do
      {:ok, %Neuron.Response{body: %{"data" => %{"getMetric" => %{"timeseriesData" => data}}}}} ->
        data

      _ ->
        get_metric(metric)
    end
  end

  defp get_metric2({metric, field}) do
    Neuron.Config.set(url: "https://api.santiment.net/graphql")
    Neuron.Config.set(connection_opts: [recv_timeout: 30_000])

    data =
      Neuron.query("""
      {
        #{metric}(slug: "#{@slug}", from: "#{@from}", to: "#{@to}", interval: "1d") {
          datetime
          #{field}
        }
      }
      """)

    case data do
      {:ok, %Neuron.Response{body: %{"data" => %{^metric => data}}}} ->
        data

      _ ->
        get_metric(metric)
    end
  end
end