Skip to content

Instantly share code, notes, and snippets.

@shaneholloman
Forked from ThinGuy/ubuntu-repo-size.bash
Last active October 1, 2024 04:45
Show Gist options
  • Save shaneholloman/125f4eaf447e31aed02e08a68d4b1938 to your computer and use it in GitHub Desktop.
Save shaneholloman/125f4eaf447e31aed02e08a68d4b1938 to your computer and use it in GitHub Desktop.
Get Ubuntu Repo Size - Non standard repos are a WIP
#!/bin/bash
ubuntu-repo-size() {
ubuntu-repo-size_usage() {
printf "\n\e[1m\e[2G%s\e[0m\n\n" "${FUNCNAME[0]%%_*}"
printf "\e[1m\e[2GUsage\e[0m: %s [options]\n\n" "${FUNCNAME[0]%%_*}"
printf "\e[1m\e[2GOptions\e[0m:\n\n"
printf "\e[3G -a, --arch \e[28GArchitecture to display, i.e. amd64,arm64,armhf,i386,ppc64el,s390x (Default: amd64)\n"
printf "\e[3G -s, --series \e[28GRelease nicknames to get information for, (Default: current LTS)\n"
printf "\e[3G -c, --components \e[28GList of repo components to query (Default: main,universe,multiverse,restricted)\n"
printf "\e[3G -p, --pockets \e[28GList of repo pockets components to query (Default: \$series \$series-updates \$series-backports \$series-security \$series-proposed)\n"
printf "\e[3G -i, --intel-repo-uri \e[28GURI of ubuntu archive for amd64/i386 (Default: http://archive.ubuntu.com/ubuntu/dists)\n"
printf "\e[3G -P, --ports-repo-uri \e[28GURI of ubuntu archive for arm64,armhf,i386,ppc64el,s390 (Default: http://ports.ubuntu.com/ubuntu/dists)\n"
printf "\e[3G -o, --cloud \e[28GInclude Ubuntu-Cloud Openstack Archive (Default: false)\n"
printf "\e[3G -C, --canonical \e[28GInclude Canonical's Partner Archive (Default: false)\n"
printf "\e[3G -S, --scale \e[28GNumber of decimal places allowed in repository size calculation (Default: 0)\n"
printf "\e[3G -f, --footnote \e[28GDisplay Ubuntu Advantage footnotes (Default: false)\n"
printf "\e[3G -I, --iec \e[28GUse IEC binary prefixes, i.e. multiples of 1024, when reporting repo sizes (Default: Use SI decimal prefixes, i.e. powers of 10)\n"
printf "\e[3G -k, --keep \e[28GKeep repo indices after script completes (Default: false)\n"
printf "\e[3G -D, --debug \e[28GVerbose mode (Default: false)\n"
printf "\e[3G -w, --work-dir\e[28GWorking directory (Default: /tmp})\n"
printf "\e[3G -h, --help\e[28GThis message\n\n"
}
export -f ubuntu-repo-size_usage
SMODE='si' SMODEK='10^3' SMODEM='10^6' SMODEG='10^9' MB='MB' GB='GB'
SC=0 KEEP=false FN=false MAIN_FN='' UNIVERSE_FN=''
ARCHES=(amd64)
SERIES=(focal)
COMPONENTS=(main universe multiverse restricted)
POCKETS=(release updates backports security proposed)
REPO_URI_UBUNTU="http://us.archive.ubuntu.com/ubuntu/dists"
REPO_URI_PORTS="http://us.ports.ubuntu.com/ubuntu-ports/dists"
REPO_URI_CLOUD="http://ubuntu-cloud.archive.canonical.com/ubuntu/dists"
REPO_URI_PARTNER="http://archive.canonical.com/ubuntu/dists"
mapfile -t SUITES_CLOUD < <(curl -sSlL "${REPO_URI_CLOUD}" | awk -F'>|/' '/folder/{print $11}')
mapfile -t SUITES_CLOUD_O7K < <(
for ((i=0; i<${#SUITES_CLOUD[@]}; i++)); do
printf '%s/%s\n' "${REPO_URI_CLOUD}" "${SUITES_CLOUD[$i]}"
done | xargs -rn1 -P0 bash -c 'curl -sSlL "$0" | awk -vURI="${0##*/}" -F">|/" "/folder/{print URI\"/\"\$11}"'
)
mapfile -t SERIES_CLOUD < <(printf '%s\n' "${SUITES_CLOUD[@]%-*}" | sort -uV)
mapfile -t POCKETS_CLOUD < <(printf '%s\n' "${SUITES_CLOUD[@]##*-}" | sort -uV)
COMPONENTS_PARTNER=(main)
mapfile -t SUITES_PARTNER < <(curl -sSlL "${REPO_URI_PARTNER}" | awk -F'>|/' '/folder/$11 ~ /-/{print $11}' | sort -uV)
mapfile -t SERIES_PARTNER < <(printf '%s\n' "${SUITES_PARTNER[@]%-*}" | sort -uV)
mapfile -t POCKETS_PARTNER < <(printf '%s/partner\n' "${SUITES_PARTNER[@]##*-}" | sort -uV)
WARGS='--retry-connrefused --waitretry=1 --timeout=25 --tries=5 --continue --no-dns-cache -qN'
CARGS='-sSlL --connect-timeout 5 --max-time 20 --retry 5 --retry-delay 1'
WORK_DIR='/tmp'
DEBUG=false
REPO_EXT=(xz gz)
ARGS=$(getopt -o a:s:c:p:oCi:P:S:fIkdw:h --long arch:,series:,components:,pockets:,intel-repo-uri:,ports-repo-uri:,cloud,canonical,debug,scale:,iec,keep,footnote,work-dir:,help -n "${FUNCNAME[0]}" -- "$@")
eval set -- "$ARGS"
while true; do
case "$1" in
-a|--arch) IFS=',' read -ra ARCHES <<< "$2"; shift 2;;
-s|--series) IFS=',' read -ra SERIES <<< "$2"; shift 2;;
-c|--components) IFS=',' read -ra COMPONENTS <<< "$2"; shift 2;;
-p|--pockets) IFS=',' read -ra POCKETS <<< "$2"; shift 2;;
-i|--intel-repo-uri) REPO_URI_INTEL="$2"; shift 2;;
-P|--ports-repo-uri) REPO_URI_PORTS="$2"; shift 2;;
-o|--cloud) CLOUD=true; shift 1;;
-C|--canonical) PARTNER=true; shift 1;;
-S|--scale) SC="$2"; shift 2;;
-f|--footnote) FN=true; MAIN_FN='\b\e[1m\u2020\e[0m'; UNIVERSE_FN='\b\e[1m\u00A7\e[0m'; shift 1;;
-I|--iec) SMODE='iec'; SMODEK='2^10'; SMODEM='2^20'; SMODEG='2^30'; MB="MiB"; GB="GiB"; shift 1;;
-k|--keep) KEEP=true; return;;
-w|--work-dir) WORK_DIR="$2"; shift 2;;
-d|--debug) DEBUG=true; shift 1;;
--) shift; break;;
esac
done
(
tput clear
tput civis
NOW=$(date +%s)sec
[[ ${DEBUG} == true ]] && set -x
if [[ ${DEBUG} == true ]]; then
WARGS="${WARGS//-q/-}"
CARGS="${CARGS//-sS/-v}"
fi
tput clear
tput civis
SRC_CNT=0 PKG_CNT=0 SRC_SZE=0 PKG_SZE=0
TOTAL_SRC_CNT=0 TOTAL_PKG_CNT=0 TOTAL_SRC_SZE=0 TOTAL_PKG_SZE=0
GTOTAL_SRC_CNT=0 GTOTAL_PKG_CNT=0 GTOTAL_SRC_SZE=0 GTOTAL_PKG_SZE=0
printf '\e[1mUbuntu Repository Size\e[0m\n\n'
(
for w in "${ARCHES[@]}"; do
if [[ ! $w == amd64 && ! $w == i386 ]]; then
REPO_URI="${REPO_URI_PORTS}"
else
REPO_URI="${REPO_URI_UBUNTU}"
fi
for x in "${SERIES[@]}"; do
FEXT="xz"
[[ $x == trusty ]] && FEXT="gz"
for y in $(printf "%s-%s\n" "$x" "${POCKETS[@]//release/${x}}" | sed "s/${x}-${x}/${x}/g"); do
for z in "${COMPONENTS[@]}"; do
echo "${WORK_DIR}/Ubuntu-Repo_${y}_${z}_Sources-${w}.${FEXT} ${REPO_URI}/${y}/${z}/source/Sources.${FEXT}"
echo "${WORK_DIR}/Ubuntu-Repo_${y}_${z}_Packages-${w}.${FEXT} ${REPO_URI}/${y}/${z}/binary-${w}/Packages.${FEXT}"
done
done
done
done
) | xargs -rn2 -P0 bash -c 'printf 1>&2 "\r\e[2G- Downloading %s\e[K" "$(echo "$0" | awk -F_ "{print \$4\" index for \"\$2\"/\"\$3}")"; wget ${WARGS} -O "$0" "$1"'
printf "\r\e[K\r"
SWORD="release"
[[ ${#SERIES[@]} -ne 1 ]] && SWORD="releases"
AWORD="arch"
[[ ${#ARCHES[@]} -ne 1 ]] && AWORD="arches"
ACNT=${#ARCHES[@]} SCNT=${#SERIES[@]}
REPO_FILE_SZE=$(du -acb "$(find "${WORK_DIR:-/tmp}" -maxdepth 1 -type f -regextype "posix-extended" -iregex '.*Ubuntu-Repo.*(Sources|Packages).*([x|g]z)')" | awk -v scale="$SC" -v smodem="$SMODEM" '/total/{printf "%.'"${SC}"'f",$1/smodem}')
REPO_FILE_CNT=$(find "${WORK_DIR:-/tmp}" -maxdepth 1 -type f -regextype "posix-extended" -iregex '.*Ubuntu-Repo.*(Sources|Packages).*([x|g]z)' | wc -l)
printf 1>&2 "\r\e[2G- Fetched %s Package/Source indices totaling %s %s in %s\e[K" "${REPO_FILE_CNT//,}" "${REPO_FILE_SZE//,}" "$MB" "$(TZ=UTC date --date now-"${NOW}" '+%Hh:%Mm:%Ss')"
sleep 2
printf 1>&2 "\r\e[K\r"
(
HEADERS=('Series (Arch)' 'Suite' 'Component' 'Src Count' "Src Size($MB/$GB)" 'Pkg Count' "Pkg Size($MB/$GB)")
(
printf "%s\n" "${HEADERS[*]}" | paste -sd'|'
for h in "${!HEADERS[@]}"; do
printf '%.3s' $(eval printf '%.0s-' "{1..${#HEADERS[$h]}}")
printf '\n'
done | paste -sd'|'
)
for w in "${ARCHES[@]}"; do
for x in "${SERIES[@]}"; do
FEXT="xz"
[[ $x == trusty ]] && FEXT="gz"
for y in $(printf "%s-%s\n" "$x" "${POCKETS[@]//release/${x}}" | sed "s/${x}-${x}/${x}/g"); do
[[ $y == "$x" ]] && printf "\e[0;1;38;2;255;255;255m\e[1;48;2;233;84;32m%s (%s)\e[0m|||||\n" "${x^}" "$w"
for z in "${COMPONENTS[@]}"; do
RCAT="xzcat"
[[ $FEXT == gz ]] && RCAT="zcat"
printf 1>&2 "\r\e[2G- Processing %s\e[K\r" "${WORK_DIR}/Ubuntu-Repo_${y}_${z}_Sources-${w}.${FEXT}"
S=$(${RCAT} "${WORK_DIR}/Ubuntu-Repo_${y}_${z}_Sources-${w}.${FEXT}" | awk -v scale="$SC" -v smodem="$SMODEM" -v smodeg="$SMODEG" '/^Files:/,/^Checksums-Sha1/{if(/^Files:/) SCNT++;SSZE+=$2}END { printf "%'"'"'d|%'"'"'.'"${SC}"'f/%.'"$SC"'f\n",SCNT,(SSZE/smodem),(SSZE/smodeg)}')
SRC_CNT=${S%%|*}
SRC_SZE=${S##*|}
SRC_SZE_MB=${SRC_SZE%%/*}
SRC_SZE_GB=${SRC_SZE##*/}
TOTAL_SRC_CNT=$(echo "scale=${SC};${TOTAL_SRC_CNT//,}+${SRC_CNT//,}" | bc)
TOTAL_SRC_SZE=$(echo "scale=${SC};${TOTAL_SRC_SZE//,}+${SRC_SZE_MB//,}" | bc)
printf 1>&2 "\r\e[2G- Processing %s\e[K\r" "${WORK_DIR}/Ubuntu-Repo_${y}_${z}_Packages-${w}.${FEXT}"
P=$(${RCAT} "${WORK_DIR}/Ubuntu-Repo_${y}_${z}_Packages-${w}.${FEXT}" | awk -v scale="$SC" -v smodem="$SMODEM" -v smodeg="$SMODEG" -vSM="${SMODE}" '/^Size: /{PSZE+=$2;PCNT++}END { printf "%'"'"'d|%'"'"'.'"$SC"'f/%.'"$SC"'f\n",PCNT,(PSZE/smodem),(PSZE/smodeg)}')
PKG_CNT=${P%%|*}
PKG_CNT=${P%%|*}
PKG_SZE=${P##*|}
PKG_SZE_MB=${PKG_SZE%%/*}
PKG_SZE_GB=${PKG_SZE##*/}
TOTAL_PKG_CNT=$(echo "scale=${SC};${TOTAL_PKG_CNT//,}+${PKG_CNT//,}" | bc)
TOTAL_PKG_SZE=$(echo "scale=${SC};${TOTAL_PKG_SZE//,}+${PKG_SZE_MB//,}" | bc)
if [[ $y == "$x" ]]; then
printf "|%s \e[3m(release)\e[0m|" "$y"
else
printf "|%s|" "$y"
fi
printf "%s|%s|%s/%s|%s|%s/%s\e[0m\n" "$z" "$SRC_CNT" "$SRC_SZE_MB" "$SRC_SZE_GB" "$PKG_CNT" "$PKG_SZE_MB" "$PKG_SZE_GB"
done
done
HEADERS[0]="" HEADERS[1]="" HEADERS[2]=""
for h in "${!HEADERS[@]}"; do
if [[ -n "${HEADERS[$h]}" ]]; then
printf '%.3s' $(eval printf '%.0s-' "{1..${#HEADERS[$h]}}")
printf '\n'
fi
done | paste -sd'|' | sed 's/^/|||/'
printf "|%s (%s) Totals:||%s|%s/%s|%s|%s/%s\n" \
"${x^}" "$w" \
"$TOTAL_SRC_CNT" \
"$TOTAL_SRC_SZE" "$(echo "scale=${SC};${TOTAL_SRC_SZE//,}/${SMODEK}" | bc)" \
"$TOTAL_PKG_CNT" \
"$TOTAL_PKG_SZE" "$(echo "scale=${SC};${TOTAL_PKG_SZE//,}/${SMODEK}" | bc)"
GTOTAL_SRC_CNT=$(echo "scale=${SC};${GTOTAL_SRC_CNT//,}+${TOTAL_SRC_CNT//,}" | bc)
GTOTAL_SRC_SZE=$(echo "scale=${SC};${GTOTAL_SRC_SZE//,}+${TOTAL_SRC_SZE//,}" | bc)
GTOTAL_PKG_CNT=$(echo "scale=${SC};${GTOTAL_PKG_CNT//,}+${TOTAL_PKG_CNT//,}" | bc)
GTOTAL_PKG_SZE=$(echo "scale=${SC};${GTOTAL_PKG_SZE//,}+${TOTAL_PKG_SZE//,}" | bc)
TOTAL_SRC_CNT=0 TOTAL_PKG_CNT=0 TOTAL_SRC_SZE=0 TOTAL_PKG_SZE=0
done
done
HEADERS[0]="" HEADERS[1]="" HEADERS[2]=""
for h in "${!HEADERS[@]}"; do
if [[ -n "${HEADERS[$h]}" ]]; then
printf '%.3s' $(eval printf '%.0s═' "{1..${#HEADERS[$h]}}")
printf '\n'
fi
done | paste -sd'|' | sed 's/^/|||/'
printf "\e[0;1;38;2;255;255;255m\e[1;48;2;233;84;32mGrand Totals:|||%s|%s %s/%s %s|%s|%s %s/%s %s\e[0m\n" \
"$GTOTAL_SRC_CNT" \
"$GTOTAL_SRC_SZE" "$MB" "$(echo "scale=${SC};${GTOTAL_SRC_SZE//,}/${SMODEK}" | bc)" "$GB" \
"$GTOTAL_PKG_CNT" \
"$GTOTAL_PKG_SZE" "$MB" "$(echo "scale=${SC};${GTOTAL_PKG_SZE//,}/${SMODEK}" | bc)" "$GB"
) | column -ts"|" | sed -E "s/^.*release.*.main\s\s|^.*updates.*.main\s\s|^.*security.*.main\s\s/&${MAIN_FN}/g;s/^.*release.*.universe\s\s|^.*updates.*.universe\s\s|^.*security.*.universe\s\s/&${UNIVERSE_FN}/g"
echo
tput cnorm
if [[ $FN == true ]]; then
printf "\n\e[%dG\e[1m\u2020\u00A0\e[0mEligble for Extended Security Maintenance (ESM) with Ubuntu Advantage for Infrastructure\n" "$(($(printf '%s\n' "${SERIES[@]}" | wc -L)+3))"
printf "\e[%dG\e[1m\u00A7\u00A0\e[0mEligble for Extended Security Maintenance (ESM) with Ubuntu Advantage for Applications\n\n" "$(($(printf '%s\n' "${SERIES[@]}" | wc -L)+3))"
fi
)
if [[ $KEEP == false ]]; then
find "${WORK_DIR}" -maxdepth 1 -type f -regextype "posix-extended" -iregex '.*Ubuntu-Repo.*(Sources|Packages).*([x|g]z)' -delete
fi
[[ $DEBUG == true ]] && set +x
}
export -f ubuntu-repo-size
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment