A small script that provides a slight wrapper over the jspm.io API (rough state and with AI suggestions that need to be cleaned up

jspm-api.fsx

#r "nuget: FsHttp"
#r "nuget: IcedTasks"
#r "nuget: JDeck, 1.0.0-beta-006"

open System
open System.Text.Json
open System.IO
open FsHttp
open IcedTasks
open JDeck
open System.Threading
open System.Text.RegularExpressions

Fsi.disableDebugLogs()

[<Literal>]
let JSPM_API_URL = "https://api.jspm.io/"

let CACHE_PATH =
  Path.Combine(
    Environment.GetFolderPath Environment.SpecialFolder.LocalApplicationData,
    "perla",
    "v1",
    "store"
  )

let LOCAL_CACHE_PATH =
  Path.Combine(Directory.GetCurrentDirectory(), ".perla", "dependencies")

let LOCAL_CACHE_PREFIX = Path.Combine(".perla", "dependencies")

type ImportMap = {
  imports: Map<string, string>
  scopes: Map<string, Map<string, string>>
  integrity: Map<string, string>
}

/// Represents the options for caching, corresponding to 'boolean | "offline"'.
type CacheOption =
  | Enabled of bool
  | Offline

type ExportCondition =
  | Development
  | Browser
  | Module
  | Custom of string

type Provider =
  | JspmIo
  | JspmIoSystem
  | NodeModules
  | Skypack
  | JsDelivr
  | Unpkg
  | EsmSh
  | Custom of string

/// These options are gathered from the serializable properties in this interface
/// https://jspm.org/docs/generator/interfaces/GeneratorOptions.html
type GeneratorOption =
  | BaseUrl of Uri
  | MapUrl of Uri
  | RootUrl of Uri
  | InputMap of ImportMap
  | DefaultProvider of Provider
  | Providers of Map<string, string>
  | ProviderConfig of Map<string, Map<string, string>>
  | Resolutions of Map<string, string>
  | Env of Set<ExportCondition>
  | Cache of CacheOption
  | Ignore of Set<string>
  | FlattenScopes of bool
  | CombineSubPaths of bool

type DownloadProvider =
  | JspmIo
  | JsDelivr
  | Unpkg

type ExcludeOption =
  | Unused
  | Types
  | SourceMaps
  | Readme
  | License

type DownloadOption =
  | Provider of DownloadProvider
  | Exclude of Set<ExcludeOption>

type DownloadPackage = { pkgUrl: Uri; files: string array }

type DownloadResponseError = { error: string }

type DownloadResponse =
  | DownloadError of DownloadResponseError
  | DownloadSuccess of Map<string, DownloadPackage>

type GeneratorResponse = {
  staticDeps: string array
  dynamicDeps: string array
  map: ImportMap
}

let (|IsJSpm|IsEsmSh|IsJsDelivr|IsUnpkg|NotSupported|)(url: Uri) =
  match url.Host with
  | "ga.jspm.io" -> IsJSpm
  | "esm.sh" -> IsEsmSh
  | "cdn.jsdelivr.net" -> IsJsDelivr
  | "unpkg.com" -> IsUnpkg
  | host -> NotSupported host

module Provider =
  let jspmRegex = lazy (Regex("npm:((?:@[^/]+/)?[^@/]+@[^/]+)"))
  let esmRegex = lazy (Regex("\*((?:@[^/]+/)?[^@/]+@[^/]+)"))
  let jsdelivrRegex = lazy (Regex("npm/((?:@[^/]+/)?[^@/]+@[^/]+)"))
  let unpkgRegex = lazy (Regex("/((?:@[^/]+/)?[^@/]+@[^/]+)"))

  type ExtractionError = { host: string; url: Uri }

  type FilePathExtractionError =
    | UnsupportedProvider of host: string * url: Uri
    | MissingPrefix of expectedPrefix: string * url: Uri
    | InvalidPackagePath of packagePath: string * url: Uri * reason: string

  let extractFromUri(uri: Uri) =
    match uri with
    | IsJSpm ->
      let m = jspmRegex.Value.Match(uri.PathAndQuery)

      if m.Success then
        m.Groups[1].Value |> Ok
      else
        Error({ host = uri.Host; url = uri })
    | IsEsmSh ->
      let m = esmRegex.Value.Match(uri.PathAndQuery)

      if m.Success then
        m.Groups[1].Value |> Ok
      else
        Error({ host = uri.Host; url = uri })
    | IsJsDelivr ->
      let m = jsdelivrRegex.Value.Match(uri.PathAndQuery)

      if m.Success then
        m.Groups[1].Value |> Ok
      else
        Error({ host = uri.Host; url = uri })
    | IsUnpkg ->
      let m = unpkgRegex.Value.Match(uri.PathAndQuery)

      if m.Success then
        m.Groups[1].Value |> Ok
      else
        Error({ host = uri.Host; url = uri })
    | NotSupported host -> Error({ host = host; url = uri })


module GeneratorOption =
  let ofSeq options =
    let finalOptions = System.Collections.Generic.Dictionary<string, obj>()

    for option in options do
      match option with
      | BaseUrl uri -> finalOptions.TryAdd("baseUrl", uri.ToString()) |> ignore
      | MapUrl uri -> finalOptions.TryAdd("mapUrl", uri.ToString()) |> ignore
      | RootUrl value ->
        finalOptions.TryAdd("rootUrl", value.ToString()) |> ignore
      | InputMap value -> finalOptions.TryAdd("inputMap", value) |> ignore
      | DefaultProvider value ->
        finalOptions.TryAdd(
          "defaultProvider",
          match value with
          | Provider.JspmIo -> "jspm.io"
          | JspmIoSystem -> "jspm.io#system"
          | NodeModules -> "nodemodles"
          | Skypack -> "skypack"
          | Provider.JsDelivr -> "jsdelivr"
          | Provider.Unpkg -> "unpkg"
          | EsmSh -> "esm.sh"
          | Custom customProvider -> customProvider
        )
        |> ignore
      | Providers value -> finalOptions.TryAdd("providers", value) |> ignore
      | ProviderConfig value ->
        finalOptions.TryAdd("providerConfig", value) |> ignore
      | Resolutions value -> finalOptions.TryAdd("resolutions", value) |> ignore
      | Env value ->
        finalOptions.TryAdd(
          "env",
          value
          |> Set.map (function
            | Development -> "development"
            | Browser -> "browser"
            | Module -> "module"
            | ExportCondition.Custom customEnv -> customEnv)
        )
        |> ignore
      | Cache value ->
        match value with
        | Enabled enabled -> finalOptions.TryAdd("cache", enabled) |> ignore
        | Offline -> finalOptions.TryAdd("cache", "offline") |> ignore
      | Ignore value -> finalOptions.TryAdd("ignore", value) |> ignore
      | FlattenScopes value ->
        finalOptions.TryAdd("flattenScopes", value) |> ignore
      | CombineSubPaths value ->
        finalOptions.TryAdd("combineSubPaths", value) |> ignore

    finalOptions

module DownloadResponse =

  let DownloadResponseDecoder: Decoder<DownloadResponse> =
    fun res -> decode {
      let attempt = Decode.auto<Map<string, DownloadPackage>> res

      match attempt with
      | Ok success -> return DownloadSuccess success
      | Error _ ->
        let! err = Decode.auto<DownloadResponseError> res
        return DownloadError err
    }


  let cache(response: Map<string, DownloadPackage>) = cancellableTask {
    let cachePath = Directory.CreateDirectory(CACHE_PATH)
    let localCachePath = Directory.CreateDirectory(LOCAL_CACHE_PATH)

    let tasks =
      response
      |> Map.toArray
      |> Array.map(fun (package, content) -> asyncEx {
        let! token = Async.CancellationToken
        let localPkgPath = Path.Combine(localCachePath.FullName, package)
        let localPkgTarget = Path.Combine(cachePath.FullName, package)

        // Create Parent Directories
        Path.GetDirectoryName(localPkgPath)
        |> Directory.CreateDirectory
        |> ignore

        // If the local store already has the package, skip creating the symbolic link
        if Directory.Exists(localPkgPath) then
          printfn
            "Package '%s' already exists, skipping symbolic link creation."
            package
        else
          Directory.CreateSymbolicLink(localPkgPath, localPkgTarget) |> ignore

        if Directory.Exists(Path.Combine(cachePath.FullName, package)) then
          printfn "Package '%s' already exists, skipping download." package
          return ()
        else
          printfn "Downloading package '%s'..." package

          for file in content.files do
            let filePath = Path.Combine(cachePath.FullName, package, file)
            let downloadUri = Uri(content.pkgUrl, file)

            let! response =
              get(downloadUri.ToString())
              |> Config.timeoutInSeconds 10
              |> Config.cancellationToken token
              |> Request.sendAsync

            use! content = response |> Response.toStreamAsync


            Directory.CreateDirectory(
              Path.GetDirectoryName filePath |> Path.GetFullPath
            )
            |> ignore

            use file = File.OpenWrite filePath

            do! content.CopyToAsync(file, cancellationToken = token)
      })

    do! Async.Parallel tasks |> Async.Ignore

    return ()
  }

module ImportMap =

  module Required =
    let map<'T> : Decoder<Map<string, 'T>> =
      fun map -> Decode.auto<Map<string, 'T>> map

  let ImportMapDecoder: Decoder<ImportMap> =
    fun map -> decode {
      let! imports =
        map |> Optional.Property.get("imports", Required.map<string>)

      let! scopes =
        map
        |> Optional.Property.get("scopes", Required.map<Map<string, string>>)

      let! integrity =
        map |> Optional.Property.get("integrity", Required.map<string>)

      return {
        imports = defaultArg imports Map.empty
        scopes = defaultArg scopes Map.empty
        integrity = defaultArg integrity Map.empty
      }
    }

  let extractPackagesWithScopes(map: ImportMap) =
    let imports = map.imports |> Map.values

    let scopeImports = map.scopes |> Map.values |> Seq.collect Map.values

    [
      for value in [| yield! imports; yield! scopeImports |] do
        let uri = Uri(value)

        match Provider.extractFromUri uri with
        | Ok package -> package
        | Error _ -> ()
    ]
    |> Set

  let install (options: GeneratorOption seq) (packages: Set<string>) = cancellableTask {
    let url = $"{JSPM_API_URL}generate"
    let! token = CancellableTask.getCancellationToken()

    let finalOptions = GeneratorOption.ofSeq options
    finalOptions.Add("install", packages)

    let! req =
      http {
        POST url
        body
        jsonSerialize finalOptions
      }
      |> Config.cancellationToken token
      |> Request.sendTAsync

    let! response = Response.deserializeJsonTAsync<GeneratorResponse> token req
    return response
  }

  let update
    (options: GeneratorOption seq)
    (map: ImportMap)
    (packages: Set<string>)
    =
    cancellableTask {
      let url = $"{JSPM_API_URL}generate"
      let! token = CancellableTask.getCancellationToken()

      let finalOptions = GeneratorOption.ofSeq options
      finalOptions.Add("update", packages)
      finalOptions["inputMap"] <- map

      let! req =
        http {
          POST url
          body
          jsonSerialize finalOptions
        }
        |> Config.cancellationToken token
        |> Request.sendTAsync

      let! response =
        Response.deserializeJsonTAsync<GeneratorResponse> token req

      return response
    }


  let uninstall
    (options: GeneratorOption seq)
    (map: ImportMap)
    (packages: Set<string>)
    =
    cancellableTask {
      let url = $"{JSPM_API_URL}generate"
      let! token = CancellableTask.getCancellationToken()

      let finalOptions = GeneratorOption.ofSeq options
      finalOptions.Add("uninstall", packages)
      finalOptions["inputMap"] <- map

      let! req =
        http {
          POST url
          body
          jsonSerialize finalOptions
        }
        |> Config.cancellationToken token
        |> Request.sendTAsync

      let! response =
        Response.deserializeJsonTAsync<GeneratorResponse> token req

      return response
    }


  let download (options: DownloadOption seq) (map: ImportMap) = cancellableTask {
    let url = $"{JSPM_API_URL}download/"
    let! token = CancellableTask.getCancellationToken()

    let! req =
      http {
        GET url

        query [
          "packages", extractPackagesWithScopes map |> String.concat ","
          for option in options do
            match option with
            | Provider provider ->
              "provider",
              match provider with
              | JspmIo -> "jspm.io"
              | JsDelivr -> "jsdelivr"
              | Unpkg -> "unpkg"
            | Exclude excludes ->
              "exclude",
              [|
                for exclude in excludes ->
                  match exclude with
                  | Unused -> "unused"
                  | Types -> "types"
                  | SourceMaps -> "sourcemaps"
                  | Readme -> "readme"
                  | License -> "license"
              |]
              |> String.concat ","
        ]

        config_cancellationToken token
      }
      |> Request.sendTAsync

    use! response = Response.toStreamAsync req

    let options =
      JsonSerializerOptions(WriteIndented = true)
      |> Codec.useDecoder DownloadResponse.DownloadResponseDecoder

    let! response =
      JsonSerializer.DeserializeAsync<DownloadResponse>(
        response,
        options,
        cancellationToken = token
      )

    match response with
    | DownloadError err ->
      return raise(Exception $"Download failed: {err.error}")
    | DownloadSuccess response ->
      printfn "Download Success: %d packages downloaded" response.Count
      // Cache the downloaded packages
      do! DownloadResponse.cache response
      return response
  }

  let toOffline (options: DownloadOption seq) (map: ImportMap) = cancellableTask {
    let allScopedImports =
      map.scopes |> Map.values |> Seq.collect Map.toSeq |> Map.ofSeq

    let combinedImports =
      map.imports
      |> Map.fold (fun state k v -> state |> Map.add k v) allScopedImports

    let! pkgs = download options { map with imports = combinedImports }
    let localCachePrefix = $"/{LOCAL_CACHE_PREFIX.Replace('\\', '/')}"

    // Helper function to extract the file path from the original URL
    let extractFilePath(uri: Uri) =
      // Common helper to extract file path after package@version part
      let extractAfterPackage(packagePath: string) =
        if packagePath.StartsWith("@") then
          // Scoped packages: @scope/package@version/file.js -> need second slash
          let firstSlash = packagePath.IndexOf('/')

          if firstSlash >= 0 then
            let afterScope = packagePath.Substring(firstSlash + 1)
            let secondSlash = afterScope.IndexOf('/')

            if secondSlash >= 0 then
              Ok(afterScope.Substring(secondSlash + 1))
            else
              Error(
                Provider.InvalidPackagePath(
                  packagePath,
                  uri,
                  "missing file path after scoped package@version"
                )
              )
          else
            Error(
              Provider.InvalidPackagePath(
                packagePath,
                uri,
                "missing package name separator in scoped package"
              )
            )
        else
          // Regular packages: package@version/file.js -> need first slash
          let firstSlash = packagePath.IndexOf('/')

          if firstSlash >= 0 then
            Ok(packagePath.Substring(firstSlash + 1))
          else
            Error(
              Provider.InvalidPackagePath(
                packagePath,
                uri,
                "missing file path after package@version"
              )
            )

      let result =
        match uri with
        | IsJSpm ->
          // Extract after "npm:" prefix
          let pathQuery = uri.PathAndQuery
          let npmIndex = pathQuery.IndexOf("npm:")

          if npmIndex >= 0 then
            pathQuery.Substring(npmIndex + 4) |> extractAfterPackage
          else
            Error(Provider.MissingPrefix("npm:", uri))
        | IsJsDelivr ->
          // Extract after "npm/" prefix
          let pathQuery = uri.PathAndQuery
          let npmIndex = pathQuery.IndexOf("npm/")

          if npmIndex >= 0 then
            pathQuery.Substring(npmIndex + 4) |> extractAfterPackage
          else
            Error(Provider.MissingPrefix("npm/", uri))
        | IsEsmSh
        | IsUnpkg ->
          // Extract after root slash
          uri.PathAndQuery.TrimStart('/') |> extractAfterPackage
        | NotSupported host -> Error(Provider.UnsupportedProvider(host, uri))

      match result with
      | Ok filePath -> filePath
      | Error error ->
        let errorMsg =
          match error with
          | Provider.UnsupportedProvider(host, url) ->
            $"Unsupported URL provider '{host}' for URL '{url}'"
          | Provider.MissingPrefix(expectedPrefix, url) ->
            $"Unable to find '{expectedPrefix}' prefix in URL '{url}'"
          | Provider.InvalidPackagePath(packagePath, url, reason) ->
            $"Invalid package path '{packagePath}' in URL '{url}': {reason}"

        printfn "Warning: %s, keeping original URL" errorMsg
        uri.ToString()

    // Build a new imports map with local paths
    let updatedImports =
      map.imports
      |> Map.map(fun pkgName importUrl ->
        let matchingKey =
          pkgs |> Map.keys |> Seq.tryFind(fun k -> k.StartsWith(pkgName + "@"))

        match matchingKey with
        | None -> importUrl // fallback to original if not found
        | Some key ->
          let uri = Uri importUrl
          let filePath = extractFilePath uri
          // If extractFilePath returned the original URL (couldn't extract), keep it as is
          if filePath = importUrl then
            importUrl
          else
            Path.Combine(localCachePrefix, key, filePath).Replace('\\', '/'))

    // Helper function to extract package name from key
    let extractPackageName(key: string) =
      let parts = key.Split('@')
      if parts.Length > 2 then "@" + parts[1] else parts[0]

    // Helper function to find matching package key
    let findMatchingKey (pkgName: string) (importUrl: string) =
      pkgs
      |> Map.keys
      |> Seq.tryFind(fun k ->
        let pkgNameFromKey = extractPackageName k
        pkgNameFromKey = pkgName || importUrl.Contains(k))

    // Helper function to convert URL to local cache path
    let convertToLocalPath importUrl matchingKey =
      match matchingKey with
      | None -> importUrl
      | Some key ->
        let uri = Uri importUrl
        let filePath = extractFilePath uri
        // If extractFilePath returned the original URL (couldn't extract), keep it as is
        if filePath = importUrl then
          importUrl
        else
          Path.Combine(localCachePrefix, key, filePath).Replace('\\', '/')

    // Helper function to update a scope map
    let updateScopeMap(scopeMap: Map<string, string>) =
      scopeMap
      |> Map.map(fun pkgName importUrl ->
        let matchingKey = findMatchingKey pkgName importUrl
        convertToLocalPath importUrl matchingKey)

    let updatedScopes =
      map.scopes
      |> Seq.map(fun (KeyValue(_, scopeMap)) ->
        localCachePrefix, updateScopeMap scopeMap)
      |> Map.ofSeq

    let offlineMap = {
      map with
          imports = updatedImports
          scopes = updatedScopes
    }

    // Write the offline import map to disk
    File.WriteAllText(
      "./offline.importmap",
      JsonSerializer.Serialize(
        offlineMap,
        JsonSerializerOptions(WriteIndented = true)
      )
    )

    return offlineMap
  }

type PackageManager =

  static member install
    (
      package: string seq,
      ?options: GeneratorOption seq,
      ?cancellationToken: CancellationToken
    ) =
    let token = defaultArg cancellationToken CancellationToken.None

    ImportMap.install
      (defaultArg options Seq.empty)
      (package |> Set.ofSeq)
      token

  static member update
    (
      packages: string seq,
      map: ImportMap,
      ?options: GeneratorOption seq,
      ?cancellationToken: CancellationToken
    ) =
    let token = defaultArg cancellationToken CancellationToken.None

    ImportMap.update
      (defaultArg options Seq.empty)
      map
      (packages |> Set.ofSeq)
      token

  static member uninstall
    (
      packages: string seq,
      map: ImportMap,
      ?options: GeneratorOption seq,
      ?cancellationToken: CancellationToken
    ) =
    let token = defaultArg cancellationToken CancellationToken.None

    ImportMap.uninstall
      (defaultArg options Seq.empty)
      map
      (packages |> Set.ofSeq)
      token

  static member download
    (
      map: ImportMap,
      ?options: DownloadOption seq,
      ?cancellationToken: CancellationToken
    ) =
    let token = defaultArg cancellationToken CancellationToken.None
    ImportMap.download (defaultArg options Seq.empty) map token

  static member toOffline
    (
      map: ImportMap,
      ?options: DownloadOption seq,
      ?cancellationToken: CancellationToken
    ) =
    let token = defaultArg cancellationToken CancellationToken.None
    ImportMap.toOffline (defaultArg options Seq.empty) map token

let work = task {
  printfn "Installing packages..."

  let! genResponse = PackageManager.install [ "jquery"; "react"; "vue"; "lit" ]

  File.WriteAllText("./dependencies.json", JsonSerializer.Serialize genResponse)

  printfn "Started downloading packages... %s" (DateTime.Now.ToString "o")

  let! response = PackageManager.download genResponse.map

  printfn "Finished downloading packages... %s" (DateTime.Now.ToString "o")

  File.WriteAllText("./downloaded.json", JsonSerializer.Serialize response)

  File.WriteAllText(
    "./dependencies.importmap",
    JsonSerializer.Serialize genResponse.map
  )

  let! result = PackageManager.toOffline genResponse.map

  printfn "Offline import map created: %A" result
}

work.GetAwaiter().GetResult()