Annotates Python requirements files with metadata from PyPI

annotate_requirements.sh

#! /bin/bash -e
# Takes a python requirements file as input and annotates each line with info from PyPI as a comment,
# outputs as file_annotated. I use it for taking one site_packages and deciding which packages I want to
# reinstall into the system python.

requirements_file=$1
filename=${requirements_file%.*}
ext=${requirements_file##*.}
outfile="$filename"_annotated."$ext"

while read line; do
  search_str=${line%%=*}
  # echo "search_str: $search_str"
  desc_html=$(curl -s https://pypi.python.org/pypi/$search_str | grep -o '<meta name="description" content=".*"/>')

  # Length of first substr to strip off is 34, of end to strip off is 34 + 3
  # Easier than verifying GNU grep vs ggrep on OSX homebrew
  str=${desc_html:34:${#desc_html}-37}
  # echo "str: $str"
  echo "$line # $str" >> $outfile
done <$requirements_file