Last active
January 4, 2025 13:38
-
-
Save lambdan/401f9124ea49eede146cf7e8dc6f28d4 to your computer and use it in GitHub Desktop.
Myrient: Calculate size of folder
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # usage: | |
| # python3 myrient_size.py URL | |
| # OR | |
| # python3 myrient_size.py URLS.txt | |
| # (where URLS.txt contains 1 URL per line) | |
| from bs4 import BeautifulSoup | |
| import sys, requests | |
| import urllib.parse | |
| def size_to_bytes(size_str): | |
| size, unit = size_str.split(" ") | |
| size = float(size) | |
| if unit == "GiB": | |
| return size * (2**30) | |
| elif unit == "MiB": | |
| return size * (2**20) | |
| elif unit == "KiB": | |
| return size * (2**10) | |
| elif unit == "B": | |
| return size; | |
| else: | |
| raise ValueError(f"Unknown size unit: {unit}") | |
| def urlDecode(s): | |
| return urllib.parse.unquote(s) | |
| def fromURL(url): | |
| response = requests.get(url) | |
| soup = BeautifulSoup(response.text, 'html.parser') | |
| elements = soup.find_all("td", class_="size") | |
| totalB = 0 | |
| for e in elements: | |
| if e.text == "-": | |
| continue | |
| totalB += size_to_bytes(e.text) | |
| totalGiB = totalB / (2**30) | |
| folder_name = urlDecode(url.split("/")[-2]) | |
| print(f"{folder_name}: {totalGiB:.2f} GiB") | |
| return totalB | |
| def fromURLS(fp): | |
| lines = [] | |
| with open(fp, 'r') as f: | |
| lines = f.readlines() | |
| totalB = 0 | |
| for l in lines: | |
| l = l.strip() | |
| if l == "": | |
| continue | |
| if l.startswith("#"): | |
| continue | |
| if l.startswith("//"): | |
| continue | |
| totalB += fromURL(l) | |
| totalTiB = totalB / (2**40) | |
| print(f"Total: {totalTiB:.3f} TiB") | |
| return totalB | |
| if sys.argv[1].startswith("http"): | |
| fromURL(sys.argv[1]) | |
| else: | |
| fromURLS(sys.argv[1]) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Example using single URL:
Example using list of URLs: