Download DJI historical data - The (mis) behavior of markets : A fractal view of Risk, Ruin and Reward - Benoît Mandelbrot / published by https://github.com/dacr/code-examples-manager #080ee523-beff-42da-a5d3-dc386ad56e9e/2cb7eba09e3386085c066b73d2263f2b38e9a925

tmbom-dji-1-download.sc

// summary : Download DJI historical data - The (mis) behavior of markets : A fractal view of Risk, Ruin and Reward - Benoît Mandelbrot
// keywords : scala, stocks, data, dji, download
// publish : gist
// authors : David Crosson
// license : Apache NON-AI License Version 2.0 (https://raw.githubusercontent.com/non-ai-licenses/non-ai-licenses/main/NON-AI-APACHE2)
// id : 080ee523-beff-42da-a5d3-dc386ad56e9e
// created-on : 2021-01-30T13:09:43Z
// managed-by : https://github.com/dacr/code-examples-manager
// execution : scala ammonite script (http://ammonite.io/) - run as follow 'amm scriptname.sc'

import $ivy.`com.lihaoyi::requests:0.6.5`
import $ivy.`com.github.pathikrit::better-files:3.9.1`
import $ivy.`org.json4s::json4s-jackson:3.6.10`
import $ivy.`org.json4s::json4s-ext:3.6.10`
import better.files._
import org.json4s._
import org.json4s.ext.{JavaTimeSerializers, JavaTypesSerializers}
import org.json4s.jackson.Serialization.write

import java.nio.charset.Charset
import java.time.LocalDate
import java.time.format.DateTimeFormatter
import scala.util.Properties.envOrNone

implicit val chosenFormats = DefaultFormats.lossless ++ JavaTimeSerializers.all ++ JavaTypesSerializers.all

val djicsv = envOrNone("TMBOM_DJI_CSV").getOrElse("tmbom_dji.csv")
val djijson = envOrNone("TMBOM_DJI_JSON").getOrElse("tmbom_dji.json")
val autoRefresh = envOrNone("TMBOM_DJI_CSV_AUTO_REFRESH").getOrElse("false").toBoolean
val charset = Charset.forName("UTF-8")
val dateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd")

val djiFile = djicsv.toFile

if (autoRefresh || djiFile.notExists) {
  for {out <- djiFile.newOutputStream.autoClosed} {
    requests
      .get("https://stooq.com/q/d/l/", params = Map("s" -> "^dji", "i" -> "d"))
      .writeBytesTo(out)
  }
}

case class Entry(date:LocalDate, open:Double, high:Double, low:Double, close:Double, volume:Option[Long])
def convert(input:String):Option[Entry] = {
  input.split(",") match {
    case Array(date, open, high, low, close) =>
      val d = LocalDate.parse(date, dateFormat)
      val e = Entry(d, open.toDouble, high.toDouble, low.toDouble, close.toDouble, None)
      Some(e)
    case Array(date, open, high, low, close, volume) =>
      val d = LocalDate.parse(date, dateFormat)
      val e = Entry(d, open.toDouble, high.toDouble, low.toDouble, close.toDouble, Some(volume.toLong))
      Some(e)
    case x =>
      println("Can't process : "+x.mkString(","))
      None
  }
}

val entries = {
  djiFile
    .lineIterator
    .to(LazyList)
    .tail
    .flatMap(convert)
}

for {
  out <- djijson.toFile.newOutputStream.autoClosed
  entry <- entries
} {
  out.write(write(entry).getBytes(charset))
  out.write('\n')
}