-
-
Save shaneholloman/125f4eaf447e31aed02e08a68d4b1938 to your computer and use it in GitHub Desktop.
Get Ubuntu Repo Size - Non standard repos are a WIP
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
ubuntu-repo-size() { | |
ubuntu-repo-size_usage() { | |
printf "\n\e[1m\e[2G%s\e[0m\n\n" "${FUNCNAME[0]%%_*}" | |
printf "\e[1m\e[2GUsage\e[0m: %s [options]\n\n" "${FUNCNAME[0]%%_*}" | |
printf "\e[1m\e[2GOptions\e[0m:\n\n" | |
printf "\e[3G -a, --arch \e[28GArchitecture to display, i.e. amd64,arm64,armhf,i386,ppc64el,s390x (Default: amd64)\n" | |
printf "\e[3G -s, --series \e[28GRelease nicknames to get information for, (Default: current LTS)\n" | |
printf "\e[3G -c, --components \e[28GList of repo components to query (Default: main,universe,multiverse,restricted)\n" | |
printf "\e[3G -p, --pockets \e[28GList of repo pockets components to query (Default: \$series \$series-updates \$series-backports \$series-security \$series-proposed)\n" | |
printf "\e[3G -i, --intel-repo-uri \e[28GURI of ubuntu archive for amd64/i386 (Default: http://archive.ubuntu.com/ubuntu/dists)\n" | |
printf "\e[3G -P, --ports-repo-uri \e[28GURI of ubuntu archive for arm64,armhf,i386,ppc64el,s390 (Default: http://ports.ubuntu.com/ubuntu/dists)\n" | |
printf "\e[3G -o, --cloud \e[28GInclude Ubuntu-Cloud Openstack Archive (Default: false)\n" | |
printf "\e[3G -C, --canonical \e[28GInclude Canonical's Partner Archive (Default: false)\n" | |
printf "\e[3G -S, --scale \e[28GNumber of decimal places allowed in repository size calculation (Default: 0)\n" | |
printf "\e[3G -f, --footnote \e[28GDisplay Ubuntu Advantage footnotes (Default: false)\n" | |
printf "\e[3G -I, --iec \e[28GUse IEC binary prefixes, i.e. multiples of 1024, when reporting repo sizes (Default: Use SI decimal prefixes, i.e. powers of 10)\n" | |
printf "\e[3G -k, --keep \e[28GKeep repo indices after script completes (Default: false)\n" | |
printf "\e[3G -D, --debug \e[28GVerbose mode (Default: false)\n" | |
printf "\e[3G -w, --work-dir\e[28GWorking directory (Default: /tmp})\n" | |
printf "\e[3G -h, --help\e[28GThis message\n\n" | |
} | |
export -f ubuntu-repo-size_usage | |
SMODE='si' SMODEK='10^3' SMODEM='10^6' SMODEG='10^9' MB='MB' GB='GB' | |
SC=0 KEEP=false FN=false MAIN_FN='' UNIVERSE_FN='' | |
ARCHES=(amd64) | |
SERIES=(focal) | |
COMPONENTS=(main universe multiverse restricted) | |
POCKETS=(release updates backports security proposed) | |
REPO_URI_UBUNTU="http://us.archive.ubuntu.com/ubuntu/dists" | |
REPO_URI_PORTS="http://us.ports.ubuntu.com/ubuntu-ports/dists" | |
REPO_URI_CLOUD="http://ubuntu-cloud.archive.canonical.com/ubuntu/dists" | |
REPO_URI_PARTNER="http://archive.canonical.com/ubuntu/dists" | |
mapfile -t SUITES_CLOUD < <(curl -sSlL "${REPO_URI_CLOUD}" | awk -F'>|/' '/folder/{print $11}') | |
mapfile -t SUITES_CLOUD_O7K < <( | |
for ((i=0; i<${#SUITES_CLOUD[@]}; i++)); do | |
printf '%s/%s\n' "${REPO_URI_CLOUD}" "${SUITES_CLOUD[$i]}" | |
done | xargs -rn1 -P0 bash -c 'curl -sSlL "$0" | awk -vURI="${0##*/}" -F">|/" "/folder/{print URI\"/\"\$11}"' | |
) | |
mapfile -t SERIES_CLOUD < <(printf '%s\n' "${SUITES_CLOUD[@]%-*}" | sort -uV) | |
mapfile -t POCKETS_CLOUD < <(printf '%s\n' "${SUITES_CLOUD[@]##*-}" | sort -uV) | |
COMPONENTS_PARTNER=(main) | |
mapfile -t SUITES_PARTNER < <(curl -sSlL "${REPO_URI_PARTNER}" | awk -F'>|/' '/folder/$11 ~ /-/{print $11}' | sort -uV) | |
mapfile -t SERIES_PARTNER < <(printf '%s\n' "${SUITES_PARTNER[@]%-*}" | sort -uV) | |
mapfile -t POCKETS_PARTNER < <(printf '%s/partner\n' "${SUITES_PARTNER[@]##*-}" | sort -uV) | |
WARGS='--retry-connrefused --waitretry=1 --timeout=25 --tries=5 --continue --no-dns-cache -qN' | |
CARGS='-sSlL --connect-timeout 5 --max-time 20 --retry 5 --retry-delay 1' | |
WORK_DIR='/tmp' | |
DEBUG=false | |
REPO_EXT=(xz gz) | |
ARGS=$(getopt -o a:s:c:p:oCi:P:S:fIkdw:h --long arch:,series:,components:,pockets:,intel-repo-uri:,ports-repo-uri:,cloud,canonical,debug,scale:,iec,keep,footnote,work-dir:,help -n "${FUNCNAME[0]}" -- "$@") | |
eval set -- "$ARGS" | |
while true; do | |
case "$1" in | |
-a|--arch) IFS=',' read -ra ARCHES <<< "$2"; shift 2;; | |
-s|--series) IFS=',' read -ra SERIES <<< "$2"; shift 2;; | |
-c|--components) IFS=',' read -ra COMPONENTS <<< "$2"; shift 2;; | |
-p|--pockets) IFS=',' read -ra POCKETS <<< "$2"; shift 2;; | |
-i|--intel-repo-uri) REPO_URI_INTEL="$2"; shift 2;; | |
-P|--ports-repo-uri) REPO_URI_PORTS="$2"; shift 2;; | |
-o|--cloud) CLOUD=true; shift 1;; | |
-C|--canonical) PARTNER=true; shift 1;; | |
-S|--scale) SC="$2"; shift 2;; | |
-f|--footnote) FN=true; MAIN_FN='\b\e[1m\u2020\e[0m'; UNIVERSE_FN='\b\e[1m\u00A7\e[0m'; shift 1;; | |
-I|--iec) SMODE='iec'; SMODEK='2^10'; SMODEM='2^20'; SMODEG='2^30'; MB="MiB"; GB="GiB"; shift 1;; | |
-k|--keep) KEEP=true; return;; | |
-w|--work-dir) WORK_DIR="$2"; shift 2;; | |
-d|--debug) DEBUG=true; shift 1;; | |
--) shift; break;; | |
esac | |
done | |
( | |
tput clear | |
tput civis | |
NOW=$(date +%s)sec | |
[[ ${DEBUG} == true ]] && set -x | |
if [[ ${DEBUG} == true ]]; then | |
WARGS="${WARGS//-q/-}" | |
CARGS="${CARGS//-sS/-v}" | |
fi | |
tput clear | |
tput civis | |
SRC_CNT=0 PKG_CNT=0 SRC_SZE=0 PKG_SZE=0 | |
TOTAL_SRC_CNT=0 TOTAL_PKG_CNT=0 TOTAL_SRC_SZE=0 TOTAL_PKG_SZE=0 | |
GTOTAL_SRC_CNT=0 GTOTAL_PKG_CNT=0 GTOTAL_SRC_SZE=0 GTOTAL_PKG_SZE=0 | |
printf '\e[1mUbuntu Repository Size\e[0m\n\n' | |
( | |
for w in "${ARCHES[@]}"; do | |
if [[ ! $w == amd64 && ! $w == i386 ]]; then | |
REPO_URI="${REPO_URI_PORTS}" | |
else | |
REPO_URI="${REPO_URI_UBUNTU}" | |
fi | |
for x in "${SERIES[@]}"; do | |
FEXT="xz" | |
[[ $x == trusty ]] && FEXT="gz" | |
for y in $(printf "%s-%s\n" "$x" "${POCKETS[@]//release/${x}}" | sed "s/${x}-${x}/${x}/g"); do | |
for z in "${COMPONENTS[@]}"; do | |
echo "${WORK_DIR}/Ubuntu-Repo_${y}_${z}_Sources-${w}.${FEXT} ${REPO_URI}/${y}/${z}/source/Sources.${FEXT}" | |
echo "${WORK_DIR}/Ubuntu-Repo_${y}_${z}_Packages-${w}.${FEXT} ${REPO_URI}/${y}/${z}/binary-${w}/Packages.${FEXT}" | |
done | |
done | |
done | |
done | |
) | xargs -rn2 -P0 bash -c 'printf 1>&2 "\r\e[2G- Downloading %s\e[K" "$(echo "$0" | awk -F_ "{print \$4\" index for \"\$2\"/\"\$3}")"; wget ${WARGS} -O "$0" "$1"' | |
printf "\r\e[K\r" | |
SWORD="release" | |
[[ ${#SERIES[@]} -ne 1 ]] && SWORD="releases" | |
AWORD="arch" | |
[[ ${#ARCHES[@]} -ne 1 ]] && AWORD="arches" | |
ACNT=${#ARCHES[@]} SCNT=${#SERIES[@]} | |
REPO_FILE_SZE=$(du -acb "$(find "${WORK_DIR:-/tmp}" -maxdepth 1 -type f -regextype "posix-extended" -iregex '.*Ubuntu-Repo.*(Sources|Packages).*([x|g]z)')" | awk -v scale="$SC" -v smodem="$SMODEM" '/total/{printf "%.'"${SC}"'f",$1/smodem}') | |
REPO_FILE_CNT=$(find "${WORK_DIR:-/tmp}" -maxdepth 1 -type f -regextype "posix-extended" -iregex '.*Ubuntu-Repo.*(Sources|Packages).*([x|g]z)' | wc -l) | |
printf 1>&2 "\r\e[2G- Fetched %s Package/Source indices totaling %s %s in %s\e[K" "${REPO_FILE_CNT//,}" "${REPO_FILE_SZE//,}" "$MB" "$(TZ=UTC date --date now-"${NOW}" '+%Hh:%Mm:%Ss')" | |
sleep 2 | |
printf 1>&2 "\r\e[K\r" | |
( | |
HEADERS=('Series (Arch)' 'Suite' 'Component' 'Src Count' "Src Size($MB/$GB)" 'Pkg Count' "Pkg Size($MB/$GB)") | |
( | |
printf "%s\n" "${HEADERS[*]}" | paste -sd'|' | |
for h in "${!HEADERS[@]}"; do | |
printf '%.3s' $(eval printf '%.0s-' "{1..${#HEADERS[$h]}}") | |
printf '\n' | |
done | paste -sd'|' | |
) | |
for w in "${ARCHES[@]}"; do | |
for x in "${SERIES[@]}"; do | |
FEXT="xz" | |
[[ $x == trusty ]] && FEXT="gz" | |
for y in $(printf "%s-%s\n" "$x" "${POCKETS[@]//release/${x}}" | sed "s/${x}-${x}/${x}/g"); do | |
[[ $y == "$x" ]] && printf "\e[0;1;38;2;255;255;255m\e[1;48;2;233;84;32m%s (%s)\e[0m|||||\n" "${x^}" "$w" | |
for z in "${COMPONENTS[@]}"; do | |
RCAT="xzcat" | |
[[ $FEXT == gz ]] && RCAT="zcat" | |
printf 1>&2 "\r\e[2G- Processing %s\e[K\r" "${WORK_DIR}/Ubuntu-Repo_${y}_${z}_Sources-${w}.${FEXT}" | |
S=$(${RCAT} "${WORK_DIR}/Ubuntu-Repo_${y}_${z}_Sources-${w}.${FEXT}" | awk -v scale="$SC" -v smodem="$SMODEM" -v smodeg="$SMODEG" '/^Files:/,/^Checksums-Sha1/{if(/^Files:/) SCNT++;SSZE+=$2}END { printf "%'"'"'d|%'"'"'.'"${SC}"'f/%.'"$SC"'f\n",SCNT,(SSZE/smodem),(SSZE/smodeg)}') | |
SRC_CNT=${S%%|*} | |
SRC_SZE=${S##*|} | |
SRC_SZE_MB=${SRC_SZE%%/*} | |
SRC_SZE_GB=${SRC_SZE##*/} | |
TOTAL_SRC_CNT=$(echo "scale=${SC};${TOTAL_SRC_CNT//,}+${SRC_CNT//,}" | bc) | |
TOTAL_SRC_SZE=$(echo "scale=${SC};${TOTAL_SRC_SZE//,}+${SRC_SZE_MB//,}" | bc) | |
printf 1>&2 "\r\e[2G- Processing %s\e[K\r" "${WORK_DIR}/Ubuntu-Repo_${y}_${z}_Packages-${w}.${FEXT}" | |
P=$(${RCAT} "${WORK_DIR}/Ubuntu-Repo_${y}_${z}_Packages-${w}.${FEXT}" | awk -v scale="$SC" -v smodem="$SMODEM" -v smodeg="$SMODEG" -vSM="${SMODE}" '/^Size: /{PSZE+=$2;PCNT++}END { printf "%'"'"'d|%'"'"'.'"$SC"'f/%.'"$SC"'f\n",PCNT,(PSZE/smodem),(PSZE/smodeg)}') | |
PKG_CNT=${P%%|*} | |
PKG_CNT=${P%%|*} | |
PKG_SZE=${P##*|} | |
PKG_SZE_MB=${PKG_SZE%%/*} | |
PKG_SZE_GB=${PKG_SZE##*/} | |
TOTAL_PKG_CNT=$(echo "scale=${SC};${TOTAL_PKG_CNT//,}+${PKG_CNT//,}" | bc) | |
TOTAL_PKG_SZE=$(echo "scale=${SC};${TOTAL_PKG_SZE//,}+${PKG_SZE_MB//,}" | bc) | |
if [[ $y == "$x" ]]; then | |
printf "|%s \e[3m(release)\e[0m|" "$y" | |
else | |
printf "|%s|" "$y" | |
fi | |
printf "%s|%s|%s/%s|%s|%s/%s\e[0m\n" "$z" "$SRC_CNT" "$SRC_SZE_MB" "$SRC_SZE_GB" "$PKG_CNT" "$PKG_SZE_MB" "$PKG_SZE_GB" | |
done | |
done | |
HEADERS[0]="" HEADERS[1]="" HEADERS[2]="" | |
for h in "${!HEADERS[@]}"; do | |
if [[ -n "${HEADERS[$h]}" ]]; then | |
printf '%.3s' $(eval printf '%.0s-' "{1..${#HEADERS[$h]}}") | |
printf '\n' | |
fi | |
done | paste -sd'|' | sed 's/^/|||/' | |
printf "|%s (%s) Totals:||%s|%s/%s|%s|%s/%s\n" \ | |
"${x^}" "$w" \ | |
"$TOTAL_SRC_CNT" \ | |
"$TOTAL_SRC_SZE" "$(echo "scale=${SC};${TOTAL_SRC_SZE//,}/${SMODEK}" | bc)" \ | |
"$TOTAL_PKG_CNT" \ | |
"$TOTAL_PKG_SZE" "$(echo "scale=${SC};${TOTAL_PKG_SZE//,}/${SMODEK}" | bc)" | |
GTOTAL_SRC_CNT=$(echo "scale=${SC};${GTOTAL_SRC_CNT//,}+${TOTAL_SRC_CNT//,}" | bc) | |
GTOTAL_SRC_SZE=$(echo "scale=${SC};${GTOTAL_SRC_SZE//,}+${TOTAL_SRC_SZE//,}" | bc) | |
GTOTAL_PKG_CNT=$(echo "scale=${SC};${GTOTAL_PKG_CNT//,}+${TOTAL_PKG_CNT//,}" | bc) | |
GTOTAL_PKG_SZE=$(echo "scale=${SC};${GTOTAL_PKG_SZE//,}+${TOTAL_PKG_SZE//,}" | bc) | |
TOTAL_SRC_CNT=0 TOTAL_PKG_CNT=0 TOTAL_SRC_SZE=0 TOTAL_PKG_SZE=0 | |
done | |
done | |
HEADERS[0]="" HEADERS[1]="" HEADERS[2]="" | |
for h in "${!HEADERS[@]}"; do | |
if [[ -n "${HEADERS[$h]}" ]]; then | |
printf '%.3s' $(eval printf '%.0s═' "{1..${#HEADERS[$h]}}") | |
printf '\n' | |
fi | |
done | paste -sd'|' | sed 's/^/|||/' | |
printf "\e[0;1;38;2;255;255;255m\e[1;48;2;233;84;32mGrand Totals:|||%s|%s %s/%s %s|%s|%s %s/%s %s\e[0m\n" \ | |
"$GTOTAL_SRC_CNT" \ | |
"$GTOTAL_SRC_SZE" "$MB" "$(echo "scale=${SC};${GTOTAL_SRC_SZE//,}/${SMODEK}" | bc)" "$GB" \ | |
"$GTOTAL_PKG_CNT" \ | |
"$GTOTAL_PKG_SZE" "$MB" "$(echo "scale=${SC};${GTOTAL_PKG_SZE//,}/${SMODEK}" | bc)" "$GB" | |
) | column -ts"|" | sed -E "s/^.*release.*.main\s\s|^.*updates.*.main\s\s|^.*security.*.main\s\s/&${MAIN_FN}/g;s/^.*release.*.universe\s\s|^.*updates.*.universe\s\s|^.*security.*.universe\s\s/&${UNIVERSE_FN}/g" | |
echo | |
tput cnorm | |
if [[ $FN == true ]]; then | |
printf "\n\e[%dG\e[1m\u2020\u00A0\e[0mEligble for Extended Security Maintenance (ESM) with Ubuntu Advantage for Infrastructure\n" "$(($(printf '%s\n' "${SERIES[@]}" | wc -L)+3))" | |
printf "\e[%dG\e[1m\u00A7\u00A0\e[0mEligble for Extended Security Maintenance (ESM) with Ubuntu Advantage for Applications\n\n" "$(($(printf '%s\n' "${SERIES[@]}" | wc -L)+3))" | |
fi | |
) | |
if [[ $KEEP == false ]]; then | |
find "${WORK_DIR}" -maxdepth 1 -type f -regextype "posix-extended" -iregex '.*Ubuntu-Repo.*(Sources|Packages).*([x|g]z)' -delete | |
fi | |
[[ $DEBUG == true ]] && set +x | |
} | |
export -f ubuntu-repo-size |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment