Skip to content

Instantly share code, notes, and snippets.

@manuzhang
Created June 28, 2019 00:30
Show Gist options
  • Save manuzhang/f1493fa978b1124dd109405f9358c0d1 to your computer and use it in GitHub Desktop.
Save manuzhang/f1493fa978b1124dd109405f9358c0d1 to your computer and use it in GitHub Desktop.
Get Github README for top starred repositories of my favorite languages
package io.github.manuzhang
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, ExecutionContext, Future, blocking}
import scala.util.{Failure, Success, Try}
object GetReadme extends App with GitHub {
val languages = List(
"Scala", "Java", "Python", "JavaScript", "Go",
"C++", "HTML", "Shell", "Jupyter Notebook", "C")
implicit val executionContext = ExecutionContext.global
languages.foreach { lang =>
val f = Future {
blocking {
get("search/repositories",
params = Map("q" -> lang, "sort" -> "stars"),
headers = Map("Accept" -> "application/vnd.github.mercy-preview+json"))
}
}.flatMap { resp =>
val fs = resp.obj("items").arr.map { item =>
val fullName = item.obj("full_name").str
Future {
blocking {
fullName -> get(s"repos/$fullName/readme")
}
}.map { case (name, value) =>
Try {
name -> value.obj("content").str
} match {
case Success(kv) => kv
case Failure(_) =>
println(s"README not found for $name")
(name, "")
}
}
}
Future.sequence(fs)
}
val json = Await.result(f, Duration.Inf).collect {
case (name, content) if content.nonEmpty =>
ujson.Obj("name" -> name, "content" -> content)
}.render(indent = 2)
os.write.over(os.pwd / s"result-$lang.json", json)
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment