github-downloader.py

import requests
import os
import time
import sys
from datetime import datetime
from typing import List, Dict, Optional

class GitHubRepoDownloader:
    """
    Download semua repository dari akun GitHub dengan handling pagination dan rate limit
    """
    
    def __init__(self, username: str, token: Optional[str] = None, output_dir: str = "github_backup"):
        """
        Inisialisasi downloader
        
        Args:
            username: Nama pengguna GitHub
            token: GitHub Personal Access Token (opsional, tapi direkomendasikan)
            output_dir: Direktori penyimpanan
        """
        self.username = username
        self.token = token
        self.output_dir = output_dir
        self.base_url = "https://api.github.com"
        self.session = self._create_session()
        self.rate_limit_remaining = None
        self.rate_limit_reset = None
        
    def _create_session(self) -> requests.Session:
        """Buat session dengan headers yang tepat"""
        session = requests.Session()
        session.headers.update({
            "Accept": "application/vnd.github.v3+json",
            "User-Agent": "GitHub-Bulk-Downloader/1.0"
        })
        
        if self.token:
            session.headers.update({
                "Authorization": f"token {self.token}"
            })
            print("[✓] Menggunakan autentikasi token (rate limit: 5000/jam)")
        else:
            print("[!] Tanpa token (rate limit: 60/jam - sangat terbatas!)")
            print("    Buat token di: https://github.com/settings/tokens")
            
        return session
    
    def _check_rate_limit(self):
        """Periksa dan handle rate limit"""
        if self.rate_limit_remaining is not None and self.rate_limit_remaining <= 5:
            reset_time = datetime.fromtimestamp(self.rate_limit_reset)
            wait_seconds = (self.rate_limit_reset - time.time()) + 5
            
            if wait_seconds > 0:
                print(f"\n[!] Rate limit hampir habis! Tersisa: {self.rate_limit_remaining} request")
                print(f"[!] Reset pada: {reset_time.strftime('%H:%M:%S')}")
                print(f"[!] Menunggu {wait_seconds:.0f} detik...")
                time.sleep(wait_seconds)
    
    def _make_request(self, url: str, params: Dict = None) -> Optional[Dict]:
        """
        Membuat request dengan rate limit handling
        
        Returns:
            Response JSON atau None jika error
        """
        self._check_rate_limit()
        
        try:
            response = self.session.get(url, params=params)
            
            # Update rate limit info dari headers
            self.rate_limit_remaining = int(response.headers.get('X-RateLimit-Remaining', 0))
            self.rate_limit_reset = int(response.headers.get('X-RateLimit-Reset', 0))
            
            if response.status_code == 403 and 'rate limit' in response.text.lower():
                reset_time = datetime.fromtimestamp(self.rate_limit_reset)
                wait_seconds = (self.rate_limit_reset - time.time()) + 5
                
                print(f"\n[✗] RATE LIMIT TERCAPAI!")
                print(f"    Reset pada: {reset_time.strftime('%Y-%m-%d %H:%M:%S')}")
                print(f"    Menunggu {wait_seconds:.0f} detik...")
                time.sleep(wait_seconds)
                return self._make_request(url, params)  # Retry
                
            response.raise_for_status()
            return response.json()
            
        except requests.exceptions.RequestException as e:
            print(f"    Error: {e}")
            return None
    
    def get_all_repos(self) -> List[Dict]:
        """
        Mendapatkan SEMUA repository dengan pagination lengkap
        
        Returns:
            List semua repository
        """
        all_repos = []
        page = 1
        per_page = 100  # Maksimal 100 per page (GitHub limit)
        
        print(f"\n[*] Mengambil daftar repository dari @{self.username}...")
        print(f"[*] Menggunakan pagination (per_page={per_page})")
        
        while True:
            url = f"{self.base_url}/users/{self.username}/repos"
            params = {
                "per_page": per_page,
                "page": page,
                "type": "all",  # Semua repo (termasuk fork? kita filter nanti)
                "sort": "created"
            }
            
            print(f"[*] Mengambil halaman {page}...", end=' ')
            
            repos = self._make_request(url, params)
            
            if not repos:
                break
                
            if len(repos) == 0:
                break
                
            # Filter fork jika tidak ingin termasuk
            # original_repos = [r for r in repos if not r['fork']]
            
            all_repos.extend(repos)
            print(f"✓ {len(repos)} repository (total: {len(all_repos)})")
            
            # Cek apakah masih ada halaman berikutnya
            if len(repos) < per_page:
                break
                
            page += 1
            time.sleep(0.2)  # Jeda kecil untuk menghindari rate limit
            
        print(f"\n[✓] TOTAL DITEMUKAN: {len(all_repos)} repository\n")
        return all_repos
    
    def download_repo_zip(self, repo_name: str, repo_full_name: str) -> bool:
        """
        Download repository sebagai file zip
        
        Returns:
            True jika berhasil, False jika gagal
        """
        # Coba berbagai kemungkinan branch
        branches_to_try = ['main', 'master', 'develop', 'dev']
        
        for branch in branches_to_try:
            zip_url = f"https://github.com/{self.username}/{repo_name}/archive/refs/heads/{branch}.zip"
            
            try:
                print(f"  ↳ Mencoba branch: {branch}", end=' ')
                response = self.session.get(zip_url, stream=True)
                
                if response.status_code == 200:
                    zip_filename = os.path.join(self.output_dir, f"{repo_name}.zip")
                    
                    # Dapatkan ukuran file
                    total_size = int(response.headers.get('content-length', 0))
                    
                    # Download dengan progress
                    downloaded = 0
                    with open(zip_filename, 'wb') as f:
                        for chunk in response.iter_content(chunk_size=8192):
                            f.write(chunk)
                            downloaded += len(chunk)
                            
                            # Tampilkan progress untuk file besar
                            if total_size > 1024 * 1024:  # >1MB
                                progress = (downloaded / total_size) * 100
                                print(f"\r  ↳ Download: {progress:.1f}%", end='')
                    
                    file_size_mb = os.path.getsize(zip_filename) / (1024 * 1024)
                    print(f"\r  ✓ Berhasil: {file_size_mb:.2f} MB      ")
                    return True
                    
            except Exception as e:
                continue
        
        print(f"\r  ✗ Gagal: Semua branch tidak ditemukan      ")
        return False
    
    def download_all_repos(self):
        """
        Download semua repository dengan progress tracking
        """
        # Buat direktori output
        if not os.path.exists(self.output_dir):
            os.makedirs(self.output_dir)
        
        # Dapatkan semua repository
        repos = self.get_all_repos()
        
        if not repos:
            print("[✗] Tidak ada repository ditemukan!")
            return
        
        # Statistik
        success_count = 0
        fail_count = 0
        failed_repos = []
        
        # Download satu per satu
        for idx, repo in enumerate(repos, 1):
            repo_name = repo['name']
            repo_full_name = repo['full_name']
            is_fork = repo.get('fork', False)
            size_kb = repo.get('size', 0)
            
            # Tampilkan info repository
            print(f"\n[{idx:3d}/{len(repos)}] 📦 {repo_name}")
            print(f"     {'🔄 Fork' if is_fork else '📁 Original'} | Size: {size_kb} KB")
            print(f"     🔗 {repo['html_url']}")
            
            # Download zip
            if self.download_repo_zip(repo_name, repo_full_name):
                success_count += 1
            else:
                fail_count += 1
                failed_repos.append(repo_name)
            
            # Tampilkan sisa rate limit
            if self.rate_limit_remaining:
                print(f"     📊 Sisa rate limit: {self.rate_limit_remaining} requests")
            
            # Jeda antar download
            time.sleep(0.5)
        
        # Tampilkan ringkasan akhir
        self._print_summary(success_count, fail_count, failed_repos)
    
    def _print_summary(self, success_count: int, fail_count: int, failed_repos: List[str]):
        """Tampilkan ringkasan download"""
        print("\n" + "="*60)
        print("📊 RINGKASAN DOWNLOAD")
        print("="*60)
        print(f"✅ Berhasil: {success_count} repository")
        print(f"❌ Gagal:   {fail_count} repository")
        print(f"📁 Lokasi:  {os.path.abspath(self.output_dir)}")
        
        if self.token:
            print(f"🔑 Auth:    Token (rate limit: 5000/jam)")
        else:
            print(f"⚠️  Auth:    Tanpa token (rate limit: 60/jam)")
            print(f"   💡 Buat token di: https://github.com/settings/tokens")
        
        if failed_repos:
            print(f"\n⚠️  Repository yang gagal didownload:")
            for repo in failed_repos:
                print(f"   - {repo}")
        
        print("="*60)


def main():
    """Fungsi utama dengan antarmuka pengguna"""
    print("="*60)
    print("🚀 GITHUB BULK REPOSITORY DOWNLOADER")
    print("="*60)
    print("Fitur:")
    print("  ✓ Pagination lengkap (ambil SEMUA repo)")
    print("  ✓ Rate limit handling otomatis")
    print("  ✓ Support token untuk rate limit lebih tinggi")
    print("  ✓ Progress download per file")
    print("="*60)
    
    # Input username
    username = input("\n📌 Masukkan username GitHub: ").strip()
    if not username:
        username = "0xWhoami35"  # Default untuk testing
    
    # Input token (opsional)
    print("\n🔑 GitHub Personal Access Token (opsional)")
    print("   Token akan meningkatkan rate limit dari 60 → 5000 request/jam")
    print("   Buat token di: https://github.com/settings/tokens")
    use_token = input("   Gunakan token? (y/n): ").strip().lower()
    
    token = None
    if use_token == 'y':
        token = input("   Masukkan token: ").strip()
        if not token:
            print("   [!] Token kosong, lanjut tanpa token")
            token = None
    
    # Input output directory
    output_dir = input("\n📁 Direktori output [default: github_backup]: ").strip()
    if not output_dir:
        output_dir = "github_backup"
    
    # Konfirmasi
    print(f"\n{'='*60}")
    print(f"📋 Konfirmasi:")
    print(f"   Username: @{username}")
    print(f"   Output:   {output_dir}/")
    print(f"   Auth:     {'Token' if token else 'Tanpa token (rate limit 60/jam)'}")
    print(f"{'='*60}")
    
    confirm = input("\n▶️  Lanjutkan download? (y/n): ").strip().lower()
    if confirm != 'y':
        print("❌ Dibatalkan")
        return
    
    # Jalankan downloader
    downloader = GitHubRepoDownloader(
        username=username,
        token=token,
        output_dir=output_dir
    )
    
    try:
        downloader.download_all_repos()
    except KeyboardInterrupt:
        print("\n\n⚠️  Download dihentikan oleh user")
    except Exception as e:
        print(f"\n❌ Error tak terduga: {e}")
        import traceback
        traceback.print_exc()


if __name__ == "__main__":
    main()