getpkg bash function for retrieving an unpkg module URL and its transitive import dependencies

getpkg.sh

# Proof of Concept
#
# download unpkg locally and all imported transitive dependencies
# only tested with ?module type URLs so far, e.g.
#
#   getpkg "https://unpkg.com/lit-element@latest?module"
#
# obviously hella brittle because module specifier extraction is
# a dumb regexp.
#
# you can also abbreviate source to just the package name/version
# and specify an output folder
#
#   getpkg lit-element packages
#

unpkg_prefix="https://unpkg.com/"

# Given a URL to unpkg.com, make a request and see if a redirection comes back.
# If we get a location header, request the URL at that location and continue
# until we don't get a location header, considering the requested URL "resolved"
#
# param 1: unpkg url
resolve_unpkg_url() {
  local resolved_url=$(curl -sI -X HEAD "$1" | grep location: | sed "s/location: \///" | grep -o "\S\+")
  if test -z $resolved_url
  then
    echo "$1"
  else
    local simplified_url="$(simplify_dotted_path $unpkg_prefix$resolved_url)"
    echo "$(resolve_unpkg_url $simplified_url)"
  fi
}

# Given a URL to unpkg.com, convert it to a path within the local root.
#
# param 1: unpkg url
# param 2: local root
unpkg_url_to_path() {
  echo ${1//$unpkg_prefix/$2} | sed 's/\?.*$//'
}

# Gven a URL or path containing path navigation segments, return a version
# without them.  Two transformations performed:
# - `/./` is converted to `/`
# - `/whatever/../` converted to `/`
#
# param 1: path or URL containing /./ or /../ segments to consume
simplify_dotted_path() {
  echo $1 | sed "s/\/\.\//\//g" | sed "s/[^\/]\{1,\}\/\.\.\///g"
}

# Given an import specifier and a base unpkg URL, return a the unpkg URL for
# the imported module.
#
# param 1: import specifier
# param 2: base url
import_specifier_to_unpkg_url() {
  if [[ $1 == $unpkg_prefix* ]]
  then
    echo "$1"
  else
    echo "$(simplify_dotted_path $2/$1)" 
  fi
}

# Given JavaScript module content, return all the import specifiers.
# Note: This is a really naive regexp function that extracts all the
# string content found inside quotes following the word `from` with
# no awareness of whether the text is in an import or export from
# statement at all.
#
# param 1: javascript content
get_import_specifiers() {
  echo $1 | grep -o "from\s\+['\"][^'\"]\+" | grep -o "[^'\"]\+$" | sort | uniq
}

# Given JavaScript module content, rewrite all import specifiers so
# they are relative paths to the local/downloaded versions.  Uses the
# base URL of the current module.
#
# param 1: javascript content
# param 2: package url
# param 3: local root
localize_unpkg_import_specifiers() {
  local content=$1
  local pkg_url=$2
  local local_root=$3
  local pkg_path="$(unpkg_url_to_path $pkg_url $local_root)"
  local pkg_dir="$(dirname $pkg_path)"
  local base_url="$(dirname $pkg_url)"
  local path_to_root=$(relative_path_to_ancestor $pkg_dir $local_root)
  local import_specifiers=$(get_import_specifiers "$content")
  for import_specifier in $import_specifiers
  do
    if [[ $import_specifier == .* ]]
    then
      local specifier_path="$(import_specifier_to_unpkg_url $import_specifier $base_url)"
    else
      local specifier_path="$import_specifier"
    fi
    if [[ $specifier_path == $unpkg_prefix* ]]
    then
      local specifier_path="$(resolve_unpkg_url $specifier_path)"
    else
      continue
    fi
    local specifier_path="$(unpkg_url_to_path $specifier_path $path_to_root)"
    local specifier_path="${specifier_path//$local_root/$path_to_root}"
    content="${content//$import_specifier/$specifier_path}"
  done
  echo "$content"
}

# Generates a "../../../" prefix to navigate back to an ancestor path from a
# a descendent path.
#
# param 1: descendant
# param 2: ancestor
relative_path_to_ancestor() {
  local descendant=$1
  local ancestor=$2
  local relpath="../"
  local descendant_slash_count=$(echo $descendant | grep -o "/" | wc -l)
  local ancestor_slash_count=$(echo $ancestor | grep -o "/" | wc -l)
  local slash_difference=$(($descendant_slash_count-$ancestor_slash_count))
  for ((i=0; i < $slash_difference; i++)); do local relpath="$relpath../"; done
  echo $relpath
}

# Given an unpkg URL, download the content and place it in the given local root,
# defaulting to `./package@version/path/to/file.js`
#
# param 1: the unpkg URL
# param 2: the destination folder (default to `./`)
# param 3: indent level
getpkg() {
  local indent=$3
  if [[ -z $indent ]]; then
    local indent=""
  fi
  local pkg_url=$1
  if [[ ! $pkg_url == $unpkg_prefix* ]]; then
    local pkg_url="$unpkg_prefix$pkg_url"
  fi
  if [[ ! $pkg_url == *\?module ]]; then
    local pkg_url="$pkg_url?module"
  fi
  if test -z $2; then
    local local_root="./"
  else
    local local_root=$2
  fi
  if [[ ! $local_root == */ ]]; then
    local local_root=$local_root/
  fi
  local pkg_url="$(resolve_unpkg_url $pkg_url)"
  local base_url=$(dirname $pkg_url)
  local pkg_path=$(unpkg_url_to_path $pkg_url $local_root)
  local pkg_dir=$(dirname $pkg_path)
  local pkg_base=$(basename $pkg_path)
  local sub_path="${pkg_path//$local_root/}"
  if test -f $pkg_path; then
    echo "$indent$sub_path (already present)"
  else
    echo "$indent$sub_path ..."
    local content="$(curl -s -L $pkg_url)"
    local import_specifiers=$(get_import_specifiers "$content")
    local content="$(localize_unpkg_import_specifiers "$content" $pkg_url $local_root)"
    mkdir -p $pkg_dir
    echo "$content" > $pkg_path
    for import_specifier in $import_specifiers
    do
      if [[ $import_specifier == .* ]]; then
        local import_specifier=$(import_specifier_to_unpkg_url $import_specifier $base_url)
      fi
      if [[ ! $import_specifier == $unpkg_prefix* ]]; then
        continue
      fi 
      getpkg $import_specifier $local_root "$indent "
    done
  fi
}

See it in action: