sdss crawl, parse and persist using sorm

CrawlAndSaveSpec.scala

import models.{DB, SdssRecord}
import org.joda.time.LocalDate
import org.specs2.mutable.Specification
import play.api.libs.ws.{WS, WSResponse}
import play.api.test.WithApplication

import scala.concurrent.Future
import scala.util.{Failure, Success}
import scala.util.matching.Regex.Match

class CrawlSpec extends Specification {
  var date = new LocalDate(2015, 3, 30)
  val today = new LocalDate()

  val recordPattern =
    """("[^"]*",?){10}""".stripMargin.r

  val url = "http://xxx.yy.zzz"

  "it" should {
    "get json result" in new WithApplication {
      while (date.isBefore(today)) {
        val dateString = date.toString("yyyyMMdd")
        println(s"start for [${dateString}]")
        val eventualResponse: Future[WSResponse] = WS.url(url).withQueryString("date" -> dateString).get()

        eventualResponse onComplete {
          case Failure(t) => println(s"error=[${t.getMessage}]")
          case Success(response) => {
            val body = response.body.substring(1, response.body.length - 1)
            val matches: Iterator[Match] = recordPattern findAllMatchIn body
            var isFirst = true
            matches.foreach { record =>
              if (isFirst)
                isFirst = false
              else {
                val fields: Array[String] = record.toString.replaceAll("[\"%+-]", "").split(",")
                val rec = SdssRecord(dateString, fields)
                DB.save(rec)
              }
            }
          }
        }
        Thread.sleep(1000)
        date = date.plusDays(1)
      }
      println(s"count=${DB.query[SdssRecord].count()}")
    }

    "record should be persistent" in new WithApplication {
      val sdssRecord = DB.save(SdssRecord(new LocalDate(), "codeName", "code", 1l, 0.0d, 0.0d, 0.0d, 1l, 0.0d, 0.0d, 0.0d))

      val newRecord = DB.fetchById[SdssRecord](sdssRecord.id)
      assert(sdssRecord === newRecord)

      val fetchOne = DB.query[SdssRecord].whereEqual("date", new LocalDate()).whereEqual("code", "code").fetchOne().get
      assert(sdssRecord === fetchOne)
    }
  }
}