Download a ton of HTTPS records, then parse them

DnsHttpsAnalysis.kt

@file:OptIn(ExperimentalTime::class)

package okhttp3.dnsoverhttps

import java.util.concurrent.LinkedBlockingDeque
import kotlin.time.Clock
import kotlin.time.ExperimentalTime
import okhttp3.Call
import okhttp3.Callback
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.RequestBody.Companion.toRequestBody
import okhttp3.Response
import okhttp3.dnsoverhttps.DnsOverHttps.Companion.DNS_MESSAGE
import okio.Buffer
import okio.ByteString.Companion.decodeHex
import okio.EOFException
import okio.FileSystem
import okio.IOException
import okio.Path
import okio.Path.Companion.toPath

/**
 * This main program downloads a bunch of HTTPS records.
 */
fun main() {
  val domainsCsv = "/Users/jessewilson/Downloads/top-1m.csv".toPath()
  val recordsTxt = "/Volumes/Development/okhttp/results.txt".toPath()
  downloadAll(
    recordsTxt = recordsTxt,
    domainsCsv = domainsCsv
  )
  readAll(
    recordsTxt = recordsTxt
  )
}

fun downloadAll(recordsTxt: Path, domainsCsv: Path) {
  val domains = readDomains(domainsCsv)
  val client = OkHttpClient()
  client.dispatcher.maxRequests = 25
  client.dispatcher.maxRequestsPerHost = 25

  val results = LinkedBlockingDeque<String>()

  for (hostname in domains) {
    val call = client.newCall(
      Request.Builder()
        .url("https://cloudflare-dns.com/dns-query".toHttpUrl())
        .header("Accept", DNS_MESSAGE.toString())
        .post(DnsRecordCodec.encodeQuery(hostname, TYPE_HTTPS).toRequestBody(DNS_MESSAGE))
        .build()
    )

    call.enqueue(object : Callback {
      override fun onFailure(call: Call, e: IOException) {
        results.put("$hostname,0,$e\n")
      }

      override fun onResponse(call: Call, response: Response) {
        response.use {
          results.put(
            "$hostname,${response.code},${
              response.body.source().readByteString().hex()
            }\n"
          )
        }
      }
    })
  }


  FileSystem.SYSTEM.write(recordsTxt) {
    writeUtf8("\n${Clock.System.now()}\n")
    for (i in 0 until domains.size) {
      writeUtf8(results.take())
    }
  }
}

private fun readDomains(path: Path): List<String> {
  return buildList {
    FileSystem.SYSTEM.read(path) {
      while (true) {
        val line = readUtf8Line() ?: break
        val comma = line.indexOf(',')
        add(line.substring(comma + 1))
      }
    }
  }
}


fun readAll(recordsTxt: Path) {
  FileSystem.SYSTEM.read(recordsTxt) {
    readUtf8Line()
    readUtf8Line()
    while (true) {
      val line = readUtf8Line() ?: break
      val parts = line.split(Regex(","), 3)
      val hostname = parts[0]
      if (parts[1] == "200") {
        try {
          val dnsMessageSource = Buffer().write(parts[2].decodeHex())
          val answers = DnsMessageReader(dnsMessageSource).read().answers
            .filterIsInstance<DnsMessageReader.ResourceRecord.Https>()
          println("$hostname: $answers")
        } catch (e: EOFException) {
          println("$hostname: failed: $e")
        }
      } else {
        println("$hostname: failed: $parts[1] $parts[2]")
      }
    }
  }
}