Embed / use Zap as a resource within a scala application

build.sbt

import sbt.*
import Keys.*

import scala.language.postfixOps
import scala.sys.process.*

val bundledZapVersion = settingKey[String]("The version of zap bundled with this release.")

ThisBuild / organization      := "uk.gov.hmrc"
ThisBuild / version           := "0.1.0-SNAPSHOT"
ThisBuild / scalaVersion      := "3.3.3"
ThisBuild / bundledZapVersion := "2.15.0"

lazy val root = (project in file("."))
  .aggregate(zapRelease, zapReleaseZip)

lazy val zapRelease = (project in file("zap-release"))
  .dependsOn(zapReleaseZip)
  .settings(
    libraryDependencies += "org.typelevel" %% "cats-effect" % "3.5.4",
    scalacOptions += "-Wnonunit-statement",
    Compile / unmanagedResourceDirectories += (zapReleaseZip / Compile / resourceManaged).value
  )

lazy val zapReleaseZip = (project in file("zap-release-zip"))
  .settings(
    Compile / resourceGenerators += Def.task {
      val zip = (Compile / resourceManaged).value / "uk" / "gov" / "hmrc" / "zap" / s"ZAP_${bundledZapVersion.value}_Core.zip"
      if(!zip.exists) {
        IO.createDirectory(zip.getParentFile)
        url(s"https://github.com/zaproxy/zaproxy/releases/download/v${bundledZapVersion.value}/ZAP_${bundledZapVersion.value}_Core.zip") #> zip !
      }
      Seq(zip)
    }.taskValue
  )

GlobalZapProxy.scala

package uk.gov.hmrc.zap

import uk.gov.hmrc.zap.ZapRelease.ZapVersion

import java.io.File
import java.net.URI
import scala.sys.process._
import scala.concurrent.duration._
import cats.effect.{IO, Resource}

import cats.effect._
import cats.effect.unsafe.implicits.global
import java.util.concurrent.atomic.AtomicBoolean

object GlobalZapProxy {
  private val initialized = new AtomicBoolean(false)

  private val downloadReport: IO[Unit] = IO {
    // TODO
    //  - since we'll probably need a http api to configure zap, use it here too
    val htmlReport = new File("report.html")
    if (htmlReport.exists) then htmlReport.delete()
    (new URI(s"http://localhost:11000/OTHER/core/other/htmlreport/").toURL #> htmlReport).!
  }

  private val zapProcess = Resource.make {
    IO(ZapVersion("2.15.0").download.startProxy) // TODO should be able to configure how it starts
      <* IO.sleep(5.seconds) // temporary, since Process.run() is async and zap startup is not instant
    // TODO
    //  - wait for zap to have finished starting
    //  - configure zap via it's http api with mdtp scanners
    //  - output the zap process logs to a tmp file
  } { zapProcess =>
    downloadReport.guarantee(IO(zapProcess.close()))
  }

  def startProxy(): Unit = {
    if (initialized.compareAndSet(false, true)) {
      val (_, finalize) = zapProcess.allocated.unsafeRunSync()
      sys.addShutdownHook {
        finalize.unsafeRunSync()
        // TODO answer question:
        //  if we start the zap process like this, do we need the cats-effect stuff?
        //  or could we just do stuff in sequence so that we have no new dependencies
        //  in the core, with ZapRelease and GlobalZapRelease we could omit cats
        //  but then have another module that builds an assembly (fat jar) and that
        //  can be run by itself like ./start-zap-proxy for people to play with.
        //  Where in that context the resource management of cats-effects is needed,
        //  and there are no problems with dependencies because it's not a library.
      }
    }
  }
}

StartZapProxy.scala

//> using scala 3.4.2
//> using dep "org.typelevel::cats-effect::3.5.4"
//> using dep "uk.gov.hmrc::zaprelease:0.1.0-SNAPSHOT"

// build with a executable we can run with with this with: (need to publishLocal zaprelease though)
//
//   scala-cli --power package --assembly StartZapProxy.scala -o start-zap-proxy -f
//
// then you can just ./start-zap-proxy

import cats.effect._

import uk.gov.hmrc.zap.ZapRelease.ZapVersion

import java.io.File
import java.net.URI
import scala.sys.process._
import scala.concurrent.duration._
import cats.effect.{IO, IOApp, Resource}

// TODO move zap-cli into the sbt project

object StartZapProxy extends IOApp.Simple:

  private val downloadReport: IO[Unit] = IO {
    // TODO
    //  - since we'll probably need a http api to configure zap, use it here too
    val htmlReport = new File("report.html")
    if (htmlReport.exists) then htmlReport.delete()
    (new URI(s"http://localhost:11000/OTHER/core/other/htmlreport/").toURL #> htmlReport).!
  }

  private val zapProxy = Resource.make {
    IO(ZapVersion("2.15.0").download.startProxy)
    // TODO
    //  - wait for zap to have finished starting
    //  - configure zap via it's http api with mdtp scanners
  } { zapProcess =>
    downloadReport.guarantee(IO(zapProcess.close()))
  }

  val run =
    zapProxy.use(_ =>
      IO.println("Starting zap proxy on http://localhost:11000") *> IO.never
    ).guarantee(
      IO.println("Stopped zap proxy") *>
      IO.println("Downloaded ./report.html")
    )

ZapRelease.scala

package uk.gov.hmrc.zap

import uk.gov.hmrc.zap.ZapRelease.ZapVersion

import java.io.File
import java.net.{URI, URL}
import java.nio.file.{Files, Path}
import java.util.zip.ZipInputStream
import scala.util.Using
import scala.sys.process.*

case class ZapRelease(location: Path):
  private lazy val startupScript: File = location.resolve("zap.sh").toFile

  def startProxy: AutoCloseable = new AutoCloseable:
    startupScript.setExecutable(true)
    private val zapProcess = Process(
      s"""bash -c "trap '' INT; ${startupScript.getAbsolutePath}"""",
      location.toFile
    ).run()
    override def close(): Unit = zapProcess.destroy()

object ZapRelease:
  opaque type ZapVersion = String

  object ZapVersion:
    def apply(zapVersion: String): ZapVersion = zapVersion
    extension (zapVersion: ZapVersion)
      def download: ZapRelease = ZapRelease.download(zapVersion)

  private def zapReleaseZipURL(zapVersion: ZapVersion): URL =
    Option(getClass.getResource(s"/uk/gov/hmrc/zap/ZAP_${zapVersion}_Core.zip"))
      .getOrElse {
        new URI(s"https://github.com/zaproxy/zaproxy/releases/download/v$zapVersion/ZAP_${zapVersion}_Core.zip").toURL
      }

  private def unzipFromURL(source: URL, target: Path): Unit =
    Using(new ZipInputStream(source.openStream())): zis =>
      LazyList
        .continually(zis.getNextEntry)
        .takeWhile(_ != null)
        .foreach: zipEntry =>
          val outFile = target.resolve(zipEntry.getName)
          if (zipEntry.isDirectory)
          then
            Files.createDirectories(outFile)
          else
            Files.createDirectories(outFile.getParent)
            Using(Files.newOutputStream(outFile))(zis.transferTo)

  private val cachedZapReleases: Path = Path.of(System.getProperty("user.home")).resolve(".cache/zap-releases")

  def download(zapVersion: ZapVersion, zapReleases: Path): ZapRelease =
    val zapRelease = zapReleases.resolve(s"ZAP_${zapVersion}")
    if (Files.notExists(zapRelease))
      then unzipFromURL(zapReleaseZipURL(zapVersion), zapReleases)
    ZapRelease(zapRelease)

  def download(zapVersion: ZapVersion): ZapRelease = ZapRelease.download(zapVersion, cachedZapReleases)

written in scala 3 so not totally copy/paste-able into our tools (but easy to rewrite to be 2.13 compatible) - but gives an example of how we could embed zap in our tools and run it rather than how we do it now - having a repo that manages building/configuring a docker container for it - which brings network related issues where we have to forward ports and stuff like that

the GlobalZapProxy stuff is intended / can then be used in your tests, like:

object TestConfiguration {

  if (sys.props.get("security.assessment").contains("true")) {
    GlobalZapProxy.startProxy()
  }

}

where most of our browser journey test repos have a TestConfiguration object which loads config and stuff like that so is a good place to do this kind of init of a proxy, if you want it to be in the control of the service, or it could be done within our ui-test-runner library