Created
February 19, 2024 02:00
-
-
Save theoparis/22c5d2ccb1a6556af4eae777467a6b47 to your computer and use it in GitHub Desktop.
Git clone multithreaded progress bar with python and rich
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
from concurrent.futures import ThreadPoolExecutor | |
import pygit2 | |
from rich.console import Group | |
from rich.live import Live | |
from rich.progress import (BarColumn, DownloadColumn, Progress, TextColumn, | |
TimeRemainingColumn, TransferSpeedColumn) | |
class MyRemoteCallbacks(pygit2.RemoteCallbacks): | |
def __init__(self, progress_bar: Progress, task: int): | |
super().__init__() | |
self.progress_bar = progress_bar | |
self.task = task | |
def transfer_progress(self, stats): | |
self.progress_bar.update(self.task, | |
total=stats.total_objects, | |
completed=stats.received_objects) | |
repos = [ | |
"https://git.theos.sh/mirrors/aws-lc", | |
"https://gitlab.kitware.com/cmake/cmake", | |
"https://github.com/llvm/llvm-project", | |
"https://git.theos.sh/mirrors/musl", | |
"https://git.theos.sh/mirrors/zlib-ng", | |
"https://git.theos.sh/mirrors/zstd", "https://github.com/python/cpython", | |
"https://github.com/NetworkConfiguration/dhcpcd", | |
"https://github.com/openssh/openssh-portable", | |
"https://github.com/landley/toybox", "https://github.com/theoparis/linux", | |
"https://github.com/u-boot/u-boot", "https://github.com/riscv/opensbi.git", | |
"https://github.com/bminor/binutils-gdb" | |
] | |
def clone_repository(repo_url: str, progress: Progress): | |
repo_name = repo_url.split("/")[-1].replace(".git", "") | |
repo_path = os.path.join("work", repo_name) | |
if not os.path.exists(repo_path): | |
task = progress.add_task(f"Cloning {repo_name}", | |
total=1, | |
repo_name=repo_name) | |
remote_callbacks = MyRemoteCallbacks(progress, task) | |
repo = pygit2.clone_repository(repo_url, | |
repo_path, | |
bare=False, | |
callbacks=remote_callbacks) | |
progress.update(task, completed=1) | |
progress.log(f"Cloned {repo_name} to {repo_path}") | |
progress.stop_task(task) | |
progress.remove_task(task) | |
progress_bars = [ | |
Progress(TextColumn("Cloning [bold blue]{task.fields[repo_name]}"), | |
BarColumn(), DownloadColumn(), TransferSpeedColumn(), | |
TimeRemainingColumn()) for repo_url in repos | |
] | |
render_group = Group(*progress_bars) | |
with Live(render_group): | |
with ThreadPoolExecutor(max_workers=4) as executor: | |
list(executor.map(clone_repository, repos, render_group.renderables)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment