See python package's direct and reverse dependencies, and version changes between revisions in milliseconds

deps

#!/bin/zsh
# Read poetry.lock and display information about dependencies:
#
# * Project dependencies
# * Sub-dependencies and reverse dependencies of packages
# * Summary of updates, or change in dependency versions between two revisions of the project
#
# Author: Sarunas Nejus, 2021
# License: MIT

helpstr="
   deps           [-lq] [<package>]
   deps  diff     [<base-ref>=HEAD] [<target-ref>]
   deps  verify   [-e <extra>] [<path>, ...]

     -h, --help  -- display help

   deps  [-l]                 -- show top-level project dependencies
          -l                  -- include sub-dependencies

   deps  [-q] <package>       -- show direct and reverse dependencies for the package
          -q                  -- exit with status 0 if package is a dependency, 1 otherwise

   deps  diff [<base-ref>=HEAD]  [<target-ref>]
                              -- summarise version changes between two revisions for all dependencies
                              -- it defaults to using a dirty poetry.lock in the current worktree

   deps  verify               -- show missing and unneeded project dependencies
   deps  verify [<path>, ...] -- select python files to consider
   deps  verify [-e <extra>]  -- select extra and its dependencies defined in pyproject.toml
"
setopt extendedglob nullglob
zmodload zsh/mapfile

PS4='%F{cyan}%D{%6.}%f %F{red}%2N%f (%I-%i)[%L %e]:%_ '

R=$'\033[0m'
B=$'\033[1m'
I=$'\033[3m'
S=$'\033[9m'
RED=$'\033[1;38;5;204m'
GREEN=$'\033[1;38;5;150m'
YELLOW=$'\033[1;38;5;222m'
CYAN=$'\033[1;38;5;117m'
MAGENTA=$'\033[1;35m'
GREY=$'\033[1;38;5;243m'

LOCKFILE=${LOCKFILE:-poetry.lock}
PYPROJECT=${PYPROJECT:-pyproject.toml}

DEPS_COLOR_PAT='
  /\b.*[a-z]/s/^/\t/
  s/^[A-Z ]+$/\n  '"$B&$R"'/
  # s/=([<>])/ \1/
  s/([a-z_-]+=)([^, ]+)/\1'"$R$B$I\2$R"'/g
  # s/(rich.tables|beetcamp|beets)[^ \t]*/'"$CYAN\1$R"'/g
  s/([ ,])([<0-9.]+[^@ ]+|[*]|$)/\1'"$RED\2$R"'/g
  /[<>~^]=?/{
    /[<>~^]+/s/([<>~^][0-9.=a-z]+)/'"$YELLOW&$R"'/g
    /</!s/(>[^ @]+)/'"$GREEN&$R"'/
  }
'

msg() {
  printf >&2 '%s\n' " · $*"
}

error() {
  echo
  msg "${RED}ERROR:$R $*"$'\n'
  exit 1
}

check_exists() {
  for file in "$@"; do
    [[ -r $file ]] || error "$file" is not found in the working directory
  done
}

deps_diff() { if ((!$#)) && git diff-files --quiet "$LOCKFILE"; then msg "$B$LOCKFILE$R is no different from the committed or staged version"
    exit
  fi

  git diff -U3 --word-diff=plain --word-diff-regex='[^. \n]+' $@ $LOCKFILE |
    tr -d '\r' |
    # remove the description line
    grep -e '^(..)?(version|optional|name) = ' -e '^..package..$' -E |
    # duplicate the version line to help with reporting in the next command
    sed -r '
      /version/p;
      s/version/& green/
    ' |
    sed -nr '
      # diff the entire version instead of first and last parts separately
      s/(\[-.*)-\](\{\+.*)\+\}(.+)\[-(.*-\])\{\+(.*\+\})/\1\3\4\2\3\5/
      # get package name, make it bold and save it
      /name = /{ s///; s/.*/'"$B&$R"'/; h; }
      # get the first version line, remove the updated version and save it
      /version = /{ s///; s/\{\+.*\+\}//g; s/$/ '$'\b''->/; H; }
      # get the second version line, remove the old version and save it
      /version green = /{ s///; s/\[-.*-\]//g; H; }
      /optional = /{
        s///
        # append optional to the saved string
        H
        # take the saved string
        x
        # remove double quotes
        s/"//g
        # color all removals in red
        s/\[-([^]]*)-\]/'"$RED\1$R"'/g
        # color all additions in green
        s/\{\+([^}]+)\+\}/'"$GREEN\1$R"'/g
        s/\n/'$'\b''/g
        p
      }
    ' | {
    output=$(</dev/stdin)
    if (( $#output )); then
      echo
      {
        echo "${B}NAME\bOLD VERSION\b->\bNEW VERSION\bOPTIONAL$R"
        echo $output
      } | column -ts$'\b'
      echo
    fi
  }
}

_requires() {
  pkg_pat=$1
  sed -nr '
    # take paragraphs starting with queried package name, until next one
    /^name = "('"$pkg_pat"')"/I,/\[\[package\]\]/{
      # its dependencies section, until next empty line
      s/name = "([^"]+)"/    '"$B\1$R"' requires/p
      /\[package.dep.*/,/^$/{
        # ignore header
        //d
        # remove double quotes, curly brackets and backslashes
        s/[\\"{}]//g
        s/ =|$/ '$'\b''/
        '"$DEPS_COLOR_PAT"'
        p
      }
    }
  ' "$LOCKFILE" | column -ts$'\b'
}

_required_by() {
  pkg_pat=$1
  echo "    $B${pkg_pat%%\|*}$R is required by"
  {
    sed -nr '
      # memorise current package name
      /^name = "([^"]+)"/{ s//\1/; h; }
      /^"?('"$pkg_pat"')"? = (["{].*["}])/I{
        # take the version or markers
        s//\2/
        # s/ ([<>=]) /=\1/
        # remove quotes and curly brackets
        s/[\\"{}]//g
        # retrieve name and version from the memory
        H; x
        # split them by \b
        s/\n/ '$'\b'' /
        p
      }
    ' "$LOCKFILE" &&
      sed -rn '
        /^('"$pkg_pat"') = (.*)/{
          s//'"$project"' '$'\b'' \2/
          s/["{}]//g
          p
        }
      ' "$PYPROJECT"
  } | sed -r "$DEPS_COLOR_PAT" | column -ts$'\b'
}

long_project_deps() {
  section=${1:+$1.}dependencies
  maindeps=($(sed -rn '
    /tool.poetry.*'"$section"'/,/^\[/{
      //d
      /^python/d
      s/^([^ ]+) =.*/\1/p
    }
  ' "$PYPROJECT" | paste -sd'|'))
  _requires "$maindeps"
}

short_project_deps() {
  section=${1:+$1.}dependencies
  {
    sed '
      # remove all blank lines except ones before a new section
      s/\n+([^[])/\n\1/g
      ' -zr $PYPROJECT |
    sed -nr '
      s/poetry.dependencies/poetry.main.dependencies/
      /\[tool.poetry.*dependencies\]/,/^$/{
        /\[tool.*\.([^.-]+[.-]dependencies)\]/{
          s//\U\1/
          s/[.-]/ /
          P
          D
        }
        # skip empty lines, python version and comments
        /^($|(python ?=|#).*$)/d
        # skip lines starting with curly brace (platform etc.)
        /^ *[{]/d
        # line ends with "[" - most likely a platform / wheel definition
        /\[$/{
          $!N
          # parse the version from the wheel file
          s/^([^ ]+) = .*\/\1-([^-]+).*/\1=\2/
        }
        # remove irrelevant characters and comments
        s/[]\[{}" ]|#.+//g
        # when requirement is an object, join members with @
        s/version=//g
        # split package name and its requirements
        s/=/ '$'\b'' /

        / ([a-z]+=)/s// '$'\b'' \1/
        / ([a-z]+=)/!s/,([a-z]+=)/ '$'\b'' \1/
        p
      }
    ' | {
      output=(${(f@)"$(</dev/stdin)"})
      print -l $output
      local -A extra_by_pkg
      extra_by_pkg=(${=${${(M)output:#*extras=*}/(#b)(#s)([^ ]##)*extras=([^ ,]##)*/$match[1] $match[2]}})

      for pkg extra in ${(kv)extra_by_pkg}; do
        sed -r '
          /^name = "('"$pkg"')"/I,/\[\[package\]\]/{
          /^'"$extra"' = \["(.*[^)])\)?"\]/{
              s//\1/
              s/%2B/+/g
              s/\(|@ / '$'\b'' /g
              s/\)?", "/\n/g
              p
            }
          }
        '
      done
    }
  } |
    sed "$DEPS_COLOR_PAT" -r |
    column -L -ts$'\b'
}

project_deps() {
  if (( $#long )); then
    check_exists $PYPROJECT $LOCKFILE
    func=long_project_deps
  else
    check_exists $PYPROJECT
    func=short_project_deps
  fi
  $func
}

pkg_deps() {
  pkg=$1
  pkg_pat="$pkg|${pkg//_/-}|${pkg//-/_}"
  pkg_with_ver=$(
    sed -rn '
      /^name = "('"$pkg_pat"')"/I{ s//\1/; h; }
      /^version = "(.+)"/{
        s//\1/; H;
        x; /'"$pkg_pat"'/I{ s/\n/ /; p; q; }
      }
    ' "$LOCKFILE"
  )
  [[ -n $pkg_with_ver ]] || error "Package $B$pkg$R is not found"
  msg "Package: $B$pkg_with_ver$R"
  echo
  _requires "$pkg_pat"
  echo
  _required_by "$pkg_pat"
  echo
}

oneline_python() {
  # args: [<python-file> ...]
  # info: _unformat given .py files: remove newlines from each statement
  sed -zr '
    s/,?\n\s*([]}).])/\1/g
    s/\n\s+(and|or)/ \1/g
    s/,\s*\n\s*/, /g
    s/([[{(])\s*\n\s+/\1/g
    s/\n\s+\n*/\n/g
  ' $@ | tr '\000' '\n'
}

find_imports() {
  # args: [PACKAGE-DIR ...]
  # info: print all 3rd-party dependencies used in _PACKAGE-DIR codebase
  ##  if _PACKAGE-DIR is not given, all .py files under the current dir are analysed recursively
  local -aU stdlib imports
  stdlib=(__future__ __main__ _dummy_thread _thread abc aifc argparse array ast asynchat asyncio asyncore atexit audioop base64 bdb binascii binhex bisect builtins bz2 calendar cgi cgitb chunk cmath cmd code codecs codeop collections collections.abc colorsys compileall concurrent.futures configparser contextlib contextvars copy copyreg cProfile crypt csv ctypes curses curses.ascii curses.panel curses.textpad dataclasses datetime dbm dbm.dumb dbm.gnu dbm.ndbm decimal difflib dis distutils distutils.archive_util distutils.bcppcompiler distutils.ccompiler distutils.cmd distutils.command distutils.command.bdist distutils.command.bdist_dumb distutils.command.bdist_msi distutils.command.bdist_packager distutils.command.bdist_rpm distutils.command.bdist_wininst distutils.command.build distutils.command.build_clib distutils.command.build_ext distutils.command.build_py distutils.command.build_scripts distutils.command.check distutils.command.clean distutils.command.config distutils.command.install distutils.command.install_data distutils.command.install_headers distutils.command.install_lib distutils.command.install_scripts distutils.command.register distutils.command.sdist distutils.core distutils.cygwinccompiler distutils.debug distutils.dep_util distutils.dir_util distutils.dist distutils.errors distutils.extension distutils.fancy_getopt distutils.file_util distutils.filelist distutils.log distutils.msvccompiler distutils.spawn distutils.sysconfig distutils.text_file distutils.unixccompiler distutils.util distutils.version doctest dummy_threading email email.charset email.contentmanager email.encoders email.errors email.generator email.header email.headerregistry email.iterators email.message email.mime email.parser email.policy email.utils encodings.idna encodings.mbcs encodings.utf_8_sig ensurepip enum errno faulthandler fcntl filecmp fileinput fnmatch formatter fractions ftplib functools gc getopt getpass gettext glob grp gzip hashlib heapq hmac html html.entities html.parser http http.client http.cookiejar http.cookies http.server imaplib imghdr imp importlib importlib.abc importlib.machinery importlib.metadata importlib.resources importlib.util inspect io ipaddress itertools json json.tool keyword lib2to3 linecache locale logging logging.config logging.handlers lzma mailbox mailcap marshal math mimetypes mmap modulefinder msilib msvcrt multiprocessing multiprocessing.connection multiprocessing.dummy multiprocessing.managers multiprocessing.pool multiprocessing.shared_memory multiprocessing.sharedctypes netrc nis nntplib numbers operator optparse os os.path ossaudiodev parser pathlib pdb pickle pickletools pipes pkgutil platform plistlib poplib posix pprint profile pstats pty pwd py_compile pyclbr pydoc queue quopri random re readline reprlib resource rlcompleter runpy sched secrets select selectors shelve shlex shutil signal site smtpd smtplib sndhdr socket socketserver spwd sqlite3 ssl stat statistics string stringprep struct subprocess sunau symbol symtable sys sysconfig syslog tabnanny tarfile telnetlib tempfile termios test test.support test.support.script_helper textwrap threading time timeit tkinter tkinter.scrolledtext tkinter.tix tkinter.ttk token tokenize trace traceback tracemalloc tty turtle turtledemo types typing unicodedata unittest unittest.mock urllib urllib.error urllib.parse urllib.request urllib.response urllib.robotparser uu uuid venv warnings wave weakref webbrowser winreg winsound wsgiref wsgiref.handlers wsgiref.headers wsgiref.simple_server wsgiref.util wsgiref.validate xdrlib xml xml.dom xml.dom.minidom xml.dom.pulldom xml.etree.ElementTree xml.parsers.expat xml.parsers.expat.errors xml.parsers.expat.model xml.sax xml.sax.handler xml.sax.saxutils xml.sax.xmlreader xmlrpc.client xmlrpc.server zipapp zipfile zipimport zlib)
  local -a files
  if (( $# )); then
    files=($^@(.) ${^@/:-.}/**/*.{py,ipynb}*)
  else
    local -a src_folders
    src_folders=(${${${(M)pyproject:#*include =*}%\"*}#*\"} ${project//-/_}(:q))
    files=(${(j:|:)~src_folders}/**/*.{py,ipynb}*)
  fi
  imports=($(oneline_python $files |
    grep -ioE '^(from .* import\b|import [^ ]+$)' |
    sed -r 's/^([^ ]* |include..)//; s/([. ].*)//' |
    grep . |
    sort -u))
  print -l ${imports:|stdlib}
}

pkg_by_root_module() {
  # args:
  # info: print root module and the package it comes from
  [[ -r pyproject.toml ]] || error pyproject.toml not found in the current directory
  local virtual_env=$VIRTUAL_ENV
  [[ -n $virtual_env ]] || virtual_env=$(poetry env info -p)

  grep -H . "$virtual_env"/lib/python*/site-packages/*dist-info/top_level.txt |
    sed '
      s/.*site-packages.//
      s/-.*:/ /
      s/\([^ ]*\) \(.*\)/\2 \1/
    ' &&
    # local path dependencies
    grep include ${^${(f@)"$(grep '^/' "$virtual_env"/lib/*/site-packages/*.pth -h)"}}/pyproject.toml |
      sed -rn '
        \=.*/([^/]+)/pyprojec.*include[^"]+"([^"]+).*=s==\2 \1=p
      '
}

verify_python_dependencies() {
  # args: [-e EXTRA-NAME]  [PATH ...]
  # info: print missing and unneeded dependencies for the package in _PWD or _PATH
  ## -e   name of the extra to check
  local -a extra
  zparseopts -D -E e:=extra
  local -a IGNORE_MISSING=(
    starlette
  )
  local -a IGNORE_UNNEEDED=(
  black
    codecov
    coveralls
    flake8{,_{bugbear,comprehensions,eradicate}}
    ipython
    isort
    mypy
    notebook
    pylint
    pytest{,_{clarity,cov,loguru,randomly,sugar,xdist}}
    uvicorn
    vulture
  )
  local -A pkg_by_module
  pkg_by_module=(
    ${(L)${$(pkg_by_root_module)//[-.]/_}}
    bs4 beautifulsoup4
    bmesh         bpy
    gi            PyGObject
    git           gitpython
    mpl_toolkits  matplotlib
    open_clip     open_clip_torch
    pil           Pillow
    sklearn       scikit_learn
    skimage       scikit_image
    git           gitpython
    gitpython     git
  )

  local -a _local deps
  if (( $#extra )); then
    deps=($(sed '
      /.*'$extra[2]' = \[([^]]+).*/{
        s//\1/
        s/"//g
        s/, /\n/g
        p
      }' -znr pyproject.toml))
    else
      deps=($(deps 2>/dev/null | sed '
        /optional.*true/d
        /^(\t[^ ]+).*/s//\1/p
      ' -rn))
  fi
  deps=(${deps//-/_})
  # local root modules
  _local=(*(/) */*.py(:r:t))
  # ignore modules that match 3rd party dependency names
  _local=(${_local:|deps})

  local -a required_modules required_pkgs
  required_modules=(${(L)${$(find_imports $@):|_local}})
  required_pkgs=(${required_modules/(#m)*/${pkg_by_module[$MATCH]:-$MATCH}})

  local -a missing unneeded
  missing=(${${(uo)${required_pkgs:|deps}}:|IGNORE_MISSING})
  unneeded=(${${(uo)${deps:|required_pkgs}}:|IGNORE_UNNEEDED})
  (( ${unneeded[(I)python_multipart]} )) && (( ${deps[(I)fastapi]} )) && unneeded=(${unneeded:#python_multipart})
  (( ${unneeded[(I)boto3_stubs]} )) && (( ${required_pkgs[(I)mypy_boto*]} )) && unneeded=(${unneeded:#boto3_stubs})
  (( ${unneeded[(I)httpx]} )) && grep -q fastapi.testclient tests/**/*.py && unneeded=(${unneeded:#httpx})
  (( ${unneeded[(I)types_beautifulsoup4]} )) && (( ${required_pkgs[(I)beautifulsoup4]} )) && unneeded=(${unneeded:#types_beautifulsoup4})
  (( ${unneeded[(I)types_Flask_Cors]} )) && (( ${required_pkgs[(I)flask-cors]} )) && unneeded=(${unneeded:#types_Flask_Cors})
  (( ${unneeded[(I)types_PyYAML]} )) && (( ${required_pkgs[(I)pyyaml]} )) && unneeded=(${unneeded:#types_PyYAML})
  (( ${unneeded[(I)types_urllib3]} )) && (( ${required_pkgs[(I)urllib3]} )) && unneeded=(${unneeded:#types_urllib3})
  (( ${unneeded[(I)types_pillow]} )) && (( ${required_pkgs[(I)pillow]} )) && unneeded=(${unneeded:#types_pillow})
  (( ${unneeded[(I)types_requests]} )) && (( ${required_pkgs[(I)requests]} )) && unneeded=(${unneeded:#types_requests})
  (( ${unneeded[(I)blender_stubs]} )) && (( ${required_pkgs[(I)bpy]} )) && unneeded=(${unneeded:#blender_stubs})

  echo
  echo "${B}MISSING DEPENDENCIES$R"
  if (( $#missing )); then
    print -l $missing
  fi
  echo
  echo "${B}UNNEEDED DEPENDENCIES$R"
  if (( $#unneeded )); then
    print -l $unneeded
  fi
}

show_help() {
  sed -r '
    ### Comments
    s/-- .*/'"$GREY&$R"'/

    ### Optional arguments
    # within brackets
    s/(\W)(-?-(\w|[-])+)/\1'"$B$YELLOW\2$R"'/g

    ### Commands
    /^(  +)([_a-z][^ A-Z]*)(  +|\t| *$)/s//\1'"$B$CYAN\2$R"'\3/

    # <arg>
    /<[^>]+>/s//'"$B$MAGENTA&$R"'/g

    ### Default values
    # =arg|=ARG
    /=((\w|-)+)/s//='"$B$GREEN\1$R"'/g

    ### Punctuation
    s/(\]+)( |$)/'"$B$YELLOW\1$R"'\2/g
    s/([m ])(\[+)/\1'"$B$YELLOW\2$R"'/g

  ' <<<"$helpstr"
}

zparseopts -D -E q=quiet d=debug l=long h=help -help=help
[[ $1 == help ]] && help=help

(( $#debug )) && set -x

if (( $#help )); then
  show_help
  exit
fi

pyproject=(${(f@)mapfile[$PYPROJECT]%$'\n'})
project=${${${pyproject[(fr)name #=*]}#*[\"\']}%[\"\']*}
if (( ! $#quiet )) && [[ $1 != diff ]]; then
  msg "Project: $B$project$R"
  msg "Version: $B${${${pyproject[(fr)version #=*]}#*[\"\']}%[\"\']*}$R"
  msg "Python:  $B${${${pyproject[(fr)python #=*]}#*[\"\']}%[\"\']*}$R"
fi

if [[ $1 == diff ]]; then
  deps_diff ${@:2}
elif [[ $1 == verify ]]; then
  verify_python_dependencies ${@:2}
elif [[ $1 == _* ]]; then
  ${1#_} ${@:2}
elif (( !$# )) || (( $#long )); then
  project_deps
elif (( $#quiet )); then
  pkg_deps $@ &>/dev/null
elif (( $# )); then
  pkg_deps $@
fi

Fix: dependencies with names starting with python are not anymore ignored in the project dependencies view