Update the apt sources.list.d/* files, to point to most recent matching release

apt-sources-fixer

#!/bin/sh

# apt-sources-fixer:
# 
# * Get online, download full list of ubuntu release codenames
# * Check the contents of the APT sources.list.d/* subfolder
# * Re-enables any 3rd party sources that got disabled by the distro's do-release-upgrade
# * For each apt source, go online and see if there is a newer release for it
# * Update the apt sources.list.d/* files, to point to most recent matching release
# 
# NOTE: Only works within sources.list.d/* subfolder. So first be sure to
#       Manually move any 3rd party entries out of main sources.list file
#       and into their own separate /etc/apt/sources.list.d/*.list files!!
# 

# Copyright 2020 Dreamcat4 <[email protected]>
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
    # http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

_usage()
{
  cat <<- "EOF"
apt-sources-fixer:

* Get online, download full list of ubuntu release codenames
* Check the contents of the APT sources.list.d/* subfolder
* Re-enables any 3rd party sources that got disabled by the distro's do-release-upgrade
* Update the apt sources.list files, to use the most recent compatible release

NOTE: Only works within sources.list.d/* subfolder. So first be sure to
      Manually move any 3rd party entries out of main sources.list file
      and into their own separate /etc/apt/sources.list.d/*.list files!!

selftest   -  print the list of release names
check      -  dont do anything, just print out a report of what will change
fix        -  fix the sources.list file (requires sudo / root privelidges)

For fix command:

* A full backup of sources.d will be created automatically, before the task is run.
* Or you can to specify an alternate folder instead of your REAL sources.list.d.
  By running these 2 extra commands beforehand:

  $ sudo cp -Rfp /etc/apt/sources.list.d /path/to/my/alternate/etc/apt/sources.list.d
  $ export APT_SOURCES_LIST_D=/path/to/my/alternate/etc/apt/sources.list.d
  $ sudo apt-sources-fixer fix

help|--help|-h|-help)
  Display this message

EOF
}

_check_privelidge()
{
  if [ ! "$(id -u -n)" = "root" ]; then
    echo "error: Insufficient Permissions to run this command."
    exit 127
  fi
}

_var_init()
{
  # get list of all ubuntu releases
  _list_rel_ver_name="$(curl https://en.wikipedia.org/wiki/Ubuntu_version_history#Table_of_versions 2> /dev/null| grep "#Ubuntu_" | sed -e "s/^.*#Ubuntu_//g" -e "s/\".*$//g")"
  _list_rel_ver_name="$(echo "$_list_rel_ver_name" | sed -e "s/[_()]/ /g" -e "s/LTS//g" -e "s/[A-Za-z]* *$//g" -e "s/  */ /g" | tr '[:upper:]' '[:lower:]')"
  _list_rel_name_ver="$(echo "$_list_rel_ver_name" | awk ' { t = $1; $1 = $2; $2 = t; print; } ')"

  _list_rel_name="$(echo "$_list_rel_name_ver" | cut -d" " -f1)"
  _list_rel_ver="$(echo "$_list_rel_ver_name" | cut -d" " -f1)"


  _list_rel_name="$(echo "$_list_rel_name" | cut -d" " -f1)"
  _list_rel_ver="$(echo "$_list_rel_ver" | cut -d" " -f1)"


  # lsb_release - current release
  _rel_ver="$(lsb_release -r | cut -f2)"
  _rel_name="$(lsb_release -c | cut -f2)"

  _list_rel_name_to_current="$(echo "$_list_rel_name" | sed -e "/$_rel_name/q")"
  _list_rel_ver_to_current="$(echo "$_list_rel_ver" | sed -e "/$_rel_ver/q")"

  _list_rel_name_ver_to_current="$(echo "$_list_rel_name_ver" | sed -e "/$_rel_name $_rel_ver/q")"
  _list_rel_ver_name_to_current="$(echo "$_list_rel_ver_name" | sed -e "/$_rel_ver $_rel_name/q")"
}


_selftest()
{
  echo "Past Ubuntu Releases:"
  echo "$_list_rel_ver_name_to_current" | tail -5
  echo ""
  echo "Current Ubuntu Release:"
  echo "$_rel_ver $_rel_name"
  echo ""
  echo "Basic early self-test complete."
  echo "Now use the \"check\" command to run a proper test pass."
  echo ""
}


if [ "$APT_SOURCES_LIST_D" ]; then
  _sources_list_d="$APT_SOURCES_LIST_D"
else
  _sources_list_d="/etc/apt/sources.list.d"
fi
_backup="${_sources_list_d}.asf.backup"

if [ ! -d "$_sources_list_d" ]; then
  echo "error: Cannot find sources.list.d folder at: $_sources_list_d, or it is not a directory"
  exit 127
fi


_backup()
{

  if [ -e "$_backup" ]; then
    echo "error: Unable to backup sources.list.d, because backup folder already exists"
    echo "error: first remove the existing backup at: ${_sources_list_d}.asf.backup"
    exit 127
  fi

  cd "$_sources_list_d"
  cd ..

  echo "Performing backup..."
  echo "========================================"
  if [ "$_fix" ]; then
    cp -Rfp "$_sources_list_d" "$_backup" || _failed=true

    if [ ""$_failed ]; then
      echo "error: Unable to backup sources.list.d"
      exit 127
    fi
  fi

  if [ "$_fix" ]; then
    echo "sources.list.d backed up to: $_backup"
  else
    echo "sources.list.d will be backed up to: $_backup"
  fi
  echo ""
}



_reenable_sources()
{
  echo "Re-enabling sources..."
  echo "========================================"
  echo ""

  cd "$_sources_list_d"

  _i=0
  for _source_file in *.list; do

    if grep -q "# disabled on upgrade" $_source_file; then

      printf "$_source_file:"

      if [ "$_fix" ]; then
        sed -i -e "/# disabled on upgrade/s/^#//g" -e "s/# disabled on upgrade.*//g" $_source_file
        printf " re-enabled\n"

      else
        # sed -e "/# disabled on upgrade/s/^#//g" -e "s/# disabled on upgrade.*//g" $_source_file
        printf " will be re-enabled\n"
      fi
      _i="$(expr $_i + 1)"
      # echo ""
    fi
  done

  if [ "$_fix" ]; then
    echo "$_i Changes made."
  else
    echo "$_i Matches found."
  fi


  _i=0
  for _release in $_list_rel_name; do

    for _source_file in $(ls *-${_release}.list 2> /dev/null); do

      _source_file_new="${_source_file%-${_release}.list}.list"

      if [ "$_fix" ]; then
        sudo mv $_source_file $_source_file_new
      else
        echo "Rename: $_source_file --> $_source_file_new"
      fi

      _i="$(expr $_i + 1)"
    done

    # echo ""
  done

  if [ "$_fix" ]; then
    echo "$_i Files renamed."
  else
    echo "$_i Files to rename."
  fi

  echo ""
  echo ""
}

_apt_update()
{
    cd "$_sources_list_d"

    echo "Running: sudo apt-update...."
    echo "========================================"

    if [ "$_fix" ]; then
      _apt_update_failed_urls="$(sudo apt update 2> /dev/null | grep -i "err:" | grep -o "http[^ ]*")"

      if [ "$_apt_update_failed_urls" ]; then
        echo "Failures detected, rolling back..."

        for _url in $_apt_update_failed_urls; do
          _file_matches="$(grep --files-with-matches -R "$_url" *.list)"

          for _file in $_file_matches; do
            sudo cp -f "${_backup}/$_file" "./$_file"
          echo "Rolled back: $_file"
          done
        done
        echo ""

        _apt_update_failed_urls="$(sudo apt update 2> /dev/null | grep -i "err:")"
        if [ "$_apt_update_failed_urls" ]; then
          echo "error: We rolled back the failed sources, but apt-update still failed. Exiting."
          exit 127
        fi

      fi
    fi

    echo "done."
    echo ""
}

_check_releases()
{
echo "Checking releases..."
echo "========================================"
echo ""

cd "$_sources_list_d"

_i=0
for _release in $_list_rel_name_to_current; do
  [ "$_release" = "$_rel_name" ] && break

  unset _matches_shown

  _source_files="$(grep --files-with-matches -R "$_release" *.list)"

  if [ "$_source_files" ]; then
    # echo "Matches for release \"$_release\""
    # echo "========================================"


    _list_from_rel_to_current=$(echo "$_list_rel_name_to_current" | sed "0,/$_release/d")
  fi

  for _source_file in $_source_files; do
    # echo "Checking $_source_file ..."

    # if [ -e "${_source_file}.done" ]; then
    #   break
    # fi

    if [ "$(echo "$_done_files" | grep -o "$_source_file")" ]; then
      break
    fi

    _deb_line="$(grep -E " $_release" "$_source_file" | grep "deb " -m1)"

    if [ ! "$_deb_line" ]; then
      _deb_line="$(grep -E " $_release" "$_source_file" | grep "deb-src" -m1)"
    fi

    [ "$_deb_line" ] || break

    _repo_url="$(echo "$_deb_line" | grep -E -o "http[^ ]*")"
    [ "$_repo_url" ] || break


    # echo "  $_repo_url"


    _dists="$(curl -L "${_repo_url}/dists" 2> /dev/null)"

    _release_latest_known="$_release"
    for _next_release in $_list_from_rel_to_current; do
      _match="$(echo "$_dists" | grep -o -E "href=.${_next_release}/\"")"

      if [ "$_match" ]; then
        _release_latest_known="$_next_release"
      else
        break
      fi
    done

    if [ "$_release_latest_known" != "$_release" ]; then
      # echo "$_source_file:"
      # echo "$_release ---> $_next_release"

      if [ ! "$_matches_shown" ]; then
        echo "Matches for release \"$_release\""
        echo "========================================"
        _matches_shown=true
      fi

      echo "$_source_file: $_release ---> $_release_latest_known"

      _done_files="$_done_files
${_source_file}"

      # sed -e "s/$_release/$_next_release/g" $_source_file
      # sed -e "s/$_release/$_next_release/g" $_source_file | sed -e "/# disabled on upgrade/s/^#//g" -e "s/# disabled on upgrade.*//g"


      # sed -e "s/$_release/$_next_release/g" -e "/# disabled on upgrade/s/^#//g" -e "s/# disabled on upgrade.*//g" $_source_file
      
      if [ "$_fix" ]; then
        sed -i -e "s/$_release/$_release_latest_known/g" $_source_file
        # touch "${_source_file}.asf.done"

      else
        # sed -e "s/$_release/$_next_release/g" -e "/# disabled on upgrade/s/^#//g" -e "s/# disabled on upgrade.*//g" $_source_file
        sleep 0
        # sed -e "s/$_release/$_next_release/g" $_source_file
      fi

      _i="$(expr $_i + 1)"

    fi


  done

  # if [ "$_source_files" ]; then
  #   echo ""
  # fi

  if [ "$_matches_shown" ]; then
    echo ""
  fi

done

if [ "$_fix" ]; then
  # rm -f *.asf.done
  echo "$_i Changes made."
else
  echo "$_i Matches found."
fi
echo ""

}

_check()
{
  _backup
  _reenable_sources
  _check_releases
  _apt_update
}


_fix()
{
  _check_privelidge
  _fix=true
  _check
}



arg="$1" ; [ "$arg" ] || arg="once"
[ "$1" ] && shift


case "$arg" in
  selftest) _var_init ; _selftest  "$@"  ;;
  check)    _var_init ; _check "$@"      ;;
  fix)      _var_init ; _fix "$@"        ;;

  # once|one|1)       _once        ;;
  # on|always)        _toggle_on   ;;
  # off|0)            _toggle_off  ;;
  # forceforce)       _force_force ;;

  help|--help|-h|-help)   _usage ;;
  *)                      _usage ;;
esac