WIP script for finding usages across hmrc repos of play-frontend-hmrc components (and other metrics)

CachedMetrics.scala

//> using scala 3.3
//> using toolkit 0.6.0
//> using toolkit typelevel:0.1.29
//> using dep com.lihaoyi::pprint:0.9.0
//> using dep com.lihaoyi::requests:0.9.0
//> using dep io.circe::circe-core:0.14.10
//> using dep io.circe::circe-generic:0.14.10
//> using dep io.circe::circe-parser:0.14.10
//> using dep "org.eclipse.jgit:org.eclipse.jgit:6.10.0.202406032230-r"
//> using dep "org.eclipse.jgit:org.eclipse.jgit.ssh.apache.agent:6.10.0.202406032230-r"
//> using dep ch.qos.logback:logback-classic:1.5.12
//> using dep org.scalameta:scalameta_2.13:4.7.7
//> using javaOpt "-XX:MaxRAMPercentage=75.0"

import Repositories.Repository
import Metric.TimedResult
import fs2.Stream
import fs2.text
import fs2.io.file.{Files, Path}
import cats.effect.*
import cats.effect.implicits.concurrentParTraverseOps
import cats.syntax.all.*
import io.circe.{Decoder, Encoder}
import io.circe.generic.auto.*
import io.circe.syntax.*
import io.circe.parser.*
import org.typelevel.log4cats.Logger
import org.typelevel.log4cats.slf4j.Slf4jLogger

import scala.meta.internal.semanticdb.{Locator, MethodSignature, SymbolInformation, TypeRef, ValueSignature}
import scala.meta.internal.semanticdb.SymbolInformation.Kind.METHOD

import scala.concurrent.duration.*
import cats.effect.unsafe.IORuntime

import scala.collection.concurrent.TrieMap
import java.time.Instant
import upickle.default.*
import io.circe

import java.time.temporal.ChronoUnit
import java.time.temporal.ChronoUnit.DAYS
import scala.collection.View
import org.eclipse.jgit.api.Git
import org.slf4j.LoggerFactory
import ch.qos.logback.classic.{Level, Logger as LogbackLogger}

import java.util.concurrent.atomic.AtomicInteger
import scala.io.Source
import scala.util.Using

given Encoder[FiniteDuration] = Encoder.encodeLong.contramap(_.toMillis)
given Decoder[FiniteDuration] = Decoder.decodeLong.map(_.millis)

val customPrinter = pprint.PPrinter.BlackWhite
  .copy(additionalHandlers = { case value: FiniteDuration =>
    pprint.Tree.Literal(s"${value.toMillis}ms")
  })

import customPrinter.tokenize

class ResourceManager(using runtime: IORuntime):
  private val shutdown: Ref[IO, IO[Unit]] = Ref.unsafe(IO.unit)

  def manage[A](resource: Resource[IO, A]): A =
    resource.allocated
      .flatMap { case (a, release) =>
        shutdown.update(release *> _).map(_ => a)
      }
      .unsafeRunSync()

  private def shutdownAll: IO[Unit] =
    shutdown.getAndSet(IO.unit).flatten

object ResourceManager:
  def apply(runtime: IORuntime): Resource[IO, ResourceManager] =
    Resource.make { IO(new ResourceManager(using runtime)) } { _.shutdownAll }

def usingResourceManager[A](block: ResourceManager ?=> IO[A]): IO[A] =
  ResourceManager(IORuntime.global).use { implicit rm => block }

extension [A](resource: Resource[IO, A])(using scope: ResourceManager) def useInScope: A = scope.manage(resource)

trait Cacheable[E]:
  type CacheKey = String
  extension (e: E) def cacheKey: CacheKey

trait Metric[E: Cacheable, M](val name: String):
  implicit protected def logger: Logger[IO] = Slf4jLogger.getLoggerFromClass[IO](getClass)

  import Metric.TimedResult

  type CacheKey = Cacheable[E]#CacheKey
  type Computed = (CacheKey, TimedResult[M])

  private val cache = TrieMap.empty[CacheKey, Deferred[IO, TimedResult[M]]]

  def set(key: CacheKey, value: TimedResult[M]): IO[Unit] =
    Deferred[IO, TimedResult[M]].flatMap: deferred =>
      logger.debug(s"$name[$key] = $value") <*
        (cache.putIfAbsent(key, deferred) match
          case None => deferred.complete(value)
          case _    => IO.unit
        )

  protected def compute(entity: E)(using Dependencies): IO[M]

  def get(entity: E)(using Dependencies): IO[M] =
    Deferred[IO, TimedResult[M]].flatMap: deferred =>
      cache.putIfAbsent(entity.cacheKey, deferred) match
        case Some(existingDeferred) =>
          existingDeferred.get.map(_.value)
        case None =>
          for
            _               <- logger.debug(s"computing $name for ${entity.cacheKey}")
            monotonicStart  <- IO.monotonic
            attempt         <- compute(entity).attempt
            monotonicFinish <- IO.monotonic
            finishedInstant <- IO.realTimeInstant
            result <- attempt match
              case Right(value) =>
                val timedResult = TimedResult(finishedInstant, monotonicFinish - monotonicStart, value)
                IO.whenA(timedResult.duration.toSeconds > 30)( // todo what constitutes long should be metric specific
                  logger.warn(s"computing $name for ${entity.cacheKey} took ${timedResult.duration.toSeconds} seconds")
                ) *>
                  deferred.complete(timedResult) *> IO.pure(value)
              case Left(error) =>
                cache.remove(entity.cacheKey)
                logger.error(s"error computing $name for ${entity.cacheKey}: $error") *>
                  IO.raiseError(error)
          yield result

  def computed: Stream[IO, (CacheKey, TimedResult[M])] =
    Stream
      .fromIterator[IO](cache.iterator, 1)
      .parEvalMapUnorderedUnbounded:
        case (key, deferred) => deferred.get.map((key, _))

object Metric:
  case class TimedResult[M](
      finished: Instant,
      duration: FiniteDuration,
      value: M
  )
  def apply[E: Cacheable, M](name: String)(f: E => Dependencies ?=> IO[M]): Metric[E, M] =
    new Metric[E, M](name):
      def compute(entity: E)(using Dependencies): IO[M] = f(entity)

extension [E](entity: E)
  def metric[M](using metric: Metric[E, M], deps: Dependencies): IO[M] =
    metric.get(entity)

object JsonMetricCache:
  implicit protected def logger: Logger[IO] = Slf4jLogger.getLoggerFromClass[IO](getClass)

  def apply[E, M: Encoder: Decoder](metric: Metric[E, M]): Resource[IO, Unit] =
    val jsonCache = Path(s"${metric.name}.json")
    initFromCache(metric, jsonCache).both(saveIntoCache(metric, jsonCache)).void

  def saveIntoCache[E, M: Encoder](metric: Metric[E, M], into: Path): Resource[IO, Unit] =
    Resource.onFinalize:
      IO(metric.computed).flatMap(stream =>
        stream
          .map(_.asJson.noSpaces)
          .intersperse("\n")
          .through(text.utf8.encode)
          .through(Files[IO].writeAll(into))
          .compile
          .drain
      )

  def initFromCache[E, M: Decoder](metric: Metric[E, M], from: Path): Resource[IO, Unit] =
    Resource.eval:
      for {
        _           <- logger.info(s"Initializing ${metric.name}")
        cacheExists <- Files[IO].exists(from)
        _ <- IO.whenA(cacheExists)(
          Files[IO]
            .readUtf8Lines(from)
            .filterNot(_.isEmpty)
            .map(decode[metric.Computed](_))
            .evalTap:
              case Right((key, value)) => metric.set(key, value)
              case Left(error)         => logger.error(s"failed to decode cache entry in ${metric.name}: $error")
            .compile
            .drain
        )
      } yield ()

def using[A](resources: Resource[IO, Unit]*)(block: IO[A]): IO[A] =
  resources.parTraverse_(_.void).use(_ => block)

object Repositories:
  private val dateBeforeWhichRepoConsideredInactive = Instant.now().minus(365, DAYS)

  lazy val findAll: Seq[Repository] =
    val localCache = os.pwd / "repositories.json"
    val repositoriesJson =
      if !os.exists(localCache)
      then os.write(localCache, requests.get.stream("https://catalogue.tax.service.gov.uk/api/v2/repositories"))
      os.read(localCache)
    decode[Seq[Repository]](repositoriesJson) match
      case Right(result) => result
      case Left(error)   => throw new RuntimeException(error)

  private lazy val reposByName: Map[String, Int] = findAll.view.zipWithIndex
    .map:
      case (repo, index) => (repo.name, index)
    .toMap
  def findByName(name: String): Option[Repository] =
    reposByName
      .get(name)
      .flatMap(findAll.get(_))

  def findAllActiveScalaFrontends: View[Repository] =
    findAll.view
      .filter(_.isActive)
      .filter(_.isScalaFrontend)

  case class Repository(
      name: String,
      description: String,
      url: String,
      createdDate: Instant,
      lastActiveDate: Instant,
      isPrivate: Boolean,
      repoType: String,
      tags: Seq[String],
      owningTeams: Seq[String],
      language: Option[String],
      isArchived: Boolean,
      defaultBranch: String,
      // branchProtection: Option[BranchProtection],
      isDeprecated: Boolean,
      teamNames: Seq[String],
      prototypeName: Option[String]
      // repositoryYamlText: Option[String]
  ):
    val isActive        = !isArchived && !isDeprecated && !lastActiveDate.isBefore(dateBeforeWhichRepoConsideredInactive)
    val isScalaFrontend = language.contains("Scala") && name.endsWith("-frontend")
    val localCopy       = os.pwd / "local-git-repos" / name

  object Repository:
    given Cacheable[Repository] with
      extension (repo: Repository) def cacheKey: CacheKey = repo.name

  case class BranchProtection(
      requiresApprovingReviews: Boolean,
      dismissesStaleReviews: Boolean,
      requiresCommitSignatures: Boolean
  )
end Repositories

//----------------------------------------------------------------------------

trait Dependencies(using val scope: ResourceManager) {}

enum LocalClone:
  case Succeeded
  case Failed(error: String)
given repoLocalClone: Metric[Repository, LocalClone] =
  Metric("repo-local-clone"): repo =>
    IO.blocking:
      if (!os.exists(repo.localCopy)) then
        os.makeDir.all(repo.localCopy)
        Git
          .cloneRepository()
          .setURI(s"git@github.com:hmrc/${repo.name}.git")
          .setDirectory(repo.localCopy.toIO)
          .call()
      LocalClone.Succeeded
    .handleErrorWith: error =>
        IO.blocking:
          os.remove.all(repo.localCopy)
          LocalClone.Failed(error.getMessage)

enum ScalaVersion:
  case Known(value: Version)
  case Unknown
given repoScalaVersion: Metric[Repository, ScalaVersion] =
  Metric("repo-scala-version"): repo =>
    (for {
      localCopy <- repo.metric[LocalClone]
      version <- localCopy match
        case LocalClone.Failed(_) => IO.pure(ScalaVersion.Unknown)
        case _ =>
          IO.blocking(
            os.makeDir.all(os.pwd / "repo-scala-version")
          ) >>
            IO.blocking(
              os.call(
                cmd = ("sbt", "--error", "--batch", "print scalaVersion"),
                timeout = 30 * 1000,
                cwd = repo.localCopy,
                mergeErrIntoOut = true,
                stdout = os.pwd / "repo-scala-version" / repo.name
              )
            ) >>
            IO.blocking(
              ScalaVersion.Known(os.read.lines(os.pwd / "repo-scala-version" / repo.name).last.trim)
            )
    } yield version).handleError(_ => ScalaVersion.Unknown)

case class Version(major: Int, minor: Int, patch: Int) extends Ordered[Version]:
  def compare(that: Version): Int =
    val majorComp = major.compare(that.major)
    if (majorComp != 0)
    then majorComp
    else
      val minorComp = minor.compare(that.minor)
      if (minorComp != 0)
      then minorComp
      else patch.compare(that.patch)

object Version:
  given string2version: Conversion[String, Version] with
    override def apply(version: String): Version =
      version.split('.').map(_.toInt).toList match
        case major :: minor :: patch :: Nil => Version(major, minor, patch)
        case _                              => throw new IllegalArgumentException(s"unable to parse version number $version")

case class VersionRange(from: Version, to: Version):
  def contains(version: Version): Boolean =
    version >= from && version <= to

// https://mvnrepository.com/artifact/org.scalameta/semanticdb-scalac
val semanticdbSupport = Seq(
  "4.9.7"  -> Seq(VersionRange("2.13.11", "2.13.15"), VersionRange("2.12.16", "2.12.20"), VersionRange("2.11.12", "2.11.12")),
  "4.8.2"  -> Seq(VersionRange("2.13.1", "2.13.11"), VersionRange("2.12.9", "2.12.18"), VersionRange("2.11.12", "2.11.12")),
  "4.5.10" -> Seq(VersionRange("2.13.0", "2.13.8"), VersionRange("2.12.8", "2.12.16"), VersionRange("2.11.12", "2.11.12"))
)
def setSemanticdbVersion(scalaVersion: ScalaVersion) = scalaVersion match
  case ScalaVersion.Known(Version(3, _, _)) => "" // because it's a builtin thing from scala 3 onwards
  case ScalaVersion.Known(version) =>
    val (semanticdbVersion, _) = semanticdbSupport
      .find((_, supportedScalaVersions) => supportedScalaVersions.exists(_.contains(version)))
      .getOrElse(throw new IllegalArgumentException(s"No semanticdb version for scala version $version"))
    s"""set semanticdbVersion := "$semanticdbVersion";"""
  case _ => throw new RuntimeException("Unknown scala version")

object Sbt: // to avoid file locks using sbt cache when compiling concurrently
  val compilationConcurrency = 6 // todo this should be configurable, is it actually helping?
  val currentCache           = new AtomicInteger(0)
  def nextCacheToUse: Int    = currentCache.getAndIncrement() % compilationConcurrency

enum CompileWithSemanticDB:
  case Succeeded
  case Failed(error: String)
given repoCompileWithSemanticDB: Metric[Repository, CompileWithSemanticDB] =
  val metricName = "repo-compile-with-semantic-db"
  Metric(metricName): repo =>
    (for
      scalaVersion <- repo.metric[ScalaVersion]
      _ <- IO.blocking {
        val sbtCache = Sbt.nextCacheToUse // todo could this be refactored to a semaphore?
        os.makeDir.all(os.pwd / metricName)
        os.call(
          cmd = (
            "sbt",
            // increase ram in case it helps, though I think compiling will be CPU bound
            s"-J-XX:MaxRAMPercentage=${Math.round(50 / Sbt.compilationConcurrency)}.0",
            // prevent waiting for interactive responses
            "--batch",
            // prevent contention over dependency caches
            s"-Dsbt.global.base=~/.sbt/sbt$sbtCache",
            s"-Dsbt.ivy.home=~/.ivy2/ivy$sbtCache",
            // prevent color codes in the output
            "-Dsbt.log.noformat=true",
            // compile with semanticdb
            // TODO use a version based on the scalaVersion of project
            s"""set semanticdbEnabled := true; ${setSemanticdbVersion(scalaVersion)} clean; compile"""
          ),
          timeout = 120 * 1000,
          cwd = repo.localCopy,
          mergeErrIntoOut = true,
          stdout = os.pwd / metricName / repo.name
        )
      }
    yield CompileWithSemanticDB.Succeeded)
      .handleError(error => CompileWithSemanticDB.Failed(error.getMessage))

object PlayFrontendHmrc:
  case class ComponentUsage(template: java.nio.file.Path, component: String, variable: String, usages: Seq[String])
  private def findUsagesIn(template: String, variable: String): Seq[String] =
    s"""(?s)@?${variable}(?=\\()(?:(?=.*?\\((?!.*?\\1)(.*\\)(?!.*\\2).*))(?=.*?\\)(?!.*?\\2)(.*)).)+?.*?(?=\\1)[^(]*(?=\\2$$)""".stripMargin.r
      .findAllMatchIn(template)
      .map(_.group(0))
      .toList
  def findComponentUsages(repo: java.nio.file.Path): Seq[ComponentUsage] =
    val usages = collection.mutable.ListBuffer[ComponentUsage]()
    Locator(repo.resolve("target"))((path, result) => {
      usages ++= result.documents.flatMap((document) => {
        document.symbols
          .collect {
            case SymbolInformation(_, _, METHOD, _, displayName, signature, _, _, _, _) =>
              signature match {
                case ValueSignature(tpe: TypeRef) if tpe.symbol.matches("""^uk/gov/hmrc/(?:govuk|hmrc)frontend/views/html.*$""") =>
                  Some((document.uri, displayName, tpe.symbol))
                case MethodSignature(_, _, returnType: TypeRef)
                    if returnType.symbol.matches("""^uk/gov/hmrc/(?:govuk|hmrc)frontend/views/html.*$""") =>
                  Some((document.uri, displayName, returnType.symbol))
                case _ => None
              }
            case _ => None
          }
          .flatten
          .map {
            case (compiledPath, variable, component) => {
              val twirlPath = repo.resolve(
                compiledPath
                  .replaceFirst("^.*/main/", "app/")
                  .replaceFirst("/html/", "/")
                  .replaceFirst("template.scala", "scala.html")
              )
              val twirlContents = Using(Source.fromFile(twirlPath.toFile))(_.mkString).getOrElse("")
              ComponentUsage(
                twirlPath,
                component,
                variable,
                findUsagesIn(twirlContents, variable)
              )
            }
          }
      })
    })
    usages.toSeq
given Metric[Repository, Seq[PlayFrontendHmrc.ComponentUsage]] = Metric("play-frontend-hmrc-usages"): repo =>
  // TODO refactor this to be less inconsistent with everything
  for {
    _ <- repo.metric[CompileWithSemanticDB]
    _ <- IO.println("🔎Searching for usages of play-frontend-hmrc components")
    usages = PlayFrontendHmrc.findComponentUsages(repo.localCopy.toNIO)
    _ <- IO.println(s"🧠Found ${usages.length} usages of our components in $repo")
  } yield usages

object Compile extends IOApp.Simple:
  def run = usingResourceManager:
    LoggerFactory.getLogger("ROOT").asInstanceOf[LogbackLogger].setLevel(Level.INFO)
    // warning debug is very noisy because it's setting it globally so for deps too

    given deps: Dependencies = new Dependencies {}

//    val repos = Repositories.findAllActiveScalaFrontends.drop(60).take(102).toList
    val repos = Repositories.findAllActiveScalaFrontends.take(10).toList // todo anyone running this start smaller

    using(
      JsonMetricCache(repoLocalClone),
      JsonMetricCache(repoScalaVersion),
      JsonMetricCache(repoCompileWithSemanticDB)
    ):
      for {
        logger         <- Slf4jLogger.create[IO]
        monotonicStart <- IO.monotonic
        // todo we're using parTraverseN but should CompileWithSemanticDB use a semaphore?
//        results         <- repos.parTraverseN(Sbt.compilationConcurrency - 2)(_.metric[CompileWithSemanticDB])
        results         <- repos.parTraverseN(3)(_.metric[Seq[PlayFrontendHmrc.ComponentUsage]])
        _               <- logger.info(s"PlayFrontendHmrc.ComponentUsages results: ${tokenize(results.flatten).mkString}")
        monotonicFinish <- IO.monotonic
        elapsed = (monotonicFinish - monotonicStart)
        _ <- logger.info(s"Computing metrics took ${elapsed.toSeconds} seconds")
      } yield ()

// did 18 with concurrency of 6 in 218955 - 3m38s
// (4 minutes) / 18 = 13 seconds each, with 6 failures (so 1/3 of total), and having to clone everything
// I notice that some I might want to put on a skip list
// Some I can fixup a cached value manually
// failures were timeouts
// 215s for 20 - so 10s each, 6 a minute still
// 1141s for 100 = 10s each still

kind of output you get example:

key bits of this is the caching to file to make it easy to rerun lots of times

caching is just json lines ish, like this cache of the scala versions: (durations are slow because it uses sbt to get it, not regex on files)