build.sbt

libraryDependencies += "org.jsoup" % "jsoup" % "1.7.3"

Main.scala

import java.io.File
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import scala.io.Source
import collection.mutable


case class Decl(val index: Int,
                val cls: String,
                val name: String,
                var doc: String)
object Main {
  def main(args: Array[String]): Unit = {
    process("/lib.scala")
    process("/Css.scala")
    process("/Html.scala")
    process("/Idb.scala")
    process("/Svg.scala")
  }

  def process(s: String, verbose: Boolean = false){
    println("Processing " + s)
    val lines = Source.fromFile("src/main/resource" +s).getLines().toSeq

    val decls = getDecls(lines)
    def report() = println(decls.values.count(_.doc != "") + " : " + decls.size)
    report()
    for (decl <- decls.values){
      val path = if (Seq("trait", "object", "class").contains(decl.name)){
        s"/API/${decl.cls}.html"
      }else{
        s"/API/${decl.cls}.${decl.name}.html"
      }
      loadFile(path).foreach(document => if (decl.doc == "") decl.doc = getText(document))
    }
    report()
    def decorate(f: Document => Seq[(String, String)]) =
      for (decl <- decls.values if Seq("trait", "object", "class").contains(decl.name)){
        val path = s"/API/${decl.cls}.html"
        loadFile(path).foreach{document =>
          val methodDefs = f(document)
          for ((name, doc) <- methodDefs){
            decls.get((decl.cls, name)).foreach(decl => if (doc.length > decl.doc.length) decl.doc = doc)
          }
        }
      }

    decorate(getProperties)
    report()
    decorate(getMethods)
    report()
    decorate(getDefLists)

    if (verbose) decls.values.toSeq.sortBy(_.index).foreach(println)
    report()
    val insertions = new Array[String](lines.length)
    for (decl <- decls.values){
      insertions(decl.index) = decl.doc
    }

    val linesOut =
      for ((line, insert) <- lines zip insertions) yield {
        val out = mutable.Buffer.empty[String]
        if (insert != null && insert.trim().length > 10){
          out.append("/**")
          var current = insert.split(" ").toList
          val buffer = mutable.Buffer.empty[String]
          while (current.length > 0){

            if (buffer.map(_.length).sum + current.head.length > 70){
              out.append("  * " + buffer.mkString(" "))
              buffer.clear()
            }else if (current.length > 0){
              buffer.append(current.head)
              current = current.tail
            }
          }
          out.append("  * " + buffer.mkString(" "))
          out.append("*/")
        }
        out.append(line)
        out
      }
    writeToFile(s.drop(1), linesOut.flatten.mkString("\n"))
  }
  def writeToFile(p: String, s: String): Unit = {
    val pw = new java.io.PrintWriter(new File(p))
    try pw.write(s) finally pw.close()
  }
  def loadFile(path: String) = {
    try
      Some(Jsoup.parse(Source.fromFile("src/main/resource" +path).mkString))
    catch {case _: Exception =>
      None
    }
  }
  def getText(input: Document) = {
    import scala.collection.JavaConversions._
    input.select(".page-content > p")
         .find(_.text().length > 34)
         .fold("")(_.text())
  }

  def getProperties(input: Document) = {
    import scala.collection.JavaConversions._
    val properties =
      input.select("tr")
        .map(e => alphanumeric(e.children().head.select("td > code").text()) -> e.children.last.text())
        .filter(_._2.length > 34)
    properties
  }
  def getDefLists(input: Document) = {
    import scala.collection.JavaConversions._
    val properties =
      input.select("dt + dd")
        .map(e => alphanumeric(e.previousElementSibling().text()) -> e.text())
        .filter(_._2.length > 34)
    properties
  }
  def alphanumeric(s: String) =
    s.split("\\.")
      .lift(1)
      .getOrElse(s)
      .split("[^A-Za-z0-9_]")
      .lift(0)
      .getOrElse(s)
      .trim()

  def getMethods(input: Document) = {
    import scala.collection.JavaConversions._
    val methods =
      input.select("h3")
        .map{e =>
          var elem = e
          val header = e.text()
          var i = 0
          var p = ""
          while(elem.nextElementSibling() != null && i < 10){
            i+= 1
            elem = e.nextElementSibling()
            if (elem.tagName() == "p" && elem.text().length > 34 && p == ""){
              p = elem.text()
            }
          }

          alphanumeric(header) -> p
        }
        .filter(_._2 != "")
    methods
  }

  def getDecls(lines: Seq[String]): Map[(String, String), Decl] = {

    var cls: String = null
    val decls = mutable.Buffer.empty[Decl]
    println("Lines " + lines.length)
    val ClsRegex = "(class|object|trait) ([0-9a-zA-Z_]+).*?".r
    val DeclRegex = "(def|val|var) ([0-9a-zA-Z_]+).*?".r
    for((line, index) <- lines.zipWithIndex){
      line.trim match{
        case ClsRegex(thing, name) =>
          cls = name
          decls.append(new Decl(index, cls, thing, ""))
        case DeclRegex(_, name) =>
          decls.append(new Decl(index, cls, name, ""))
        case _ =>
      }
    }

    decls.map(d => (d.cls, d.name) -> d).toMap
  }
}