Crawler.scala

def crawlUrl(data: CrawlerData, url: Url): IO[Nothing, CrawlerData] = {
  if (!data.visitedLinks.contains(url)) {
    workerFor(data, url.host).flatMap {
      case (data2, workerQueue) =>
        workerQueue.offer(url).map { _ =>
          data2.copy(
            visitedLinks = data.visitedLinks + url,
            inProgress = data.inProgress + url
          )
        }
    }
  } else IO.now(data)
}