Gist from Drafts

download-audio.md

#!/bin/bash

# Configuratie
JSON_FILE="/Users/pad/naar/geen-NL-mannen.json"
DOWNLOAD_DIR="/Users/pad/naar/downloadmap/audio"
LOG_FILE="$DOWNLOAD_DIR/download.log"
FAILED_FILE="$DOWNLOAD_DIR/failed_downloads.txt"

# Kleuren voor output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Functie voor logging
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

# Functie voor grootte formattering
format_bytes() {
    local bytes=$1
    if [[ $bytes -lt 1024 ]]; then
        echo "${bytes}B"
    elif [[ $bytes -lt 1048576 ]]; then
        echo "$(( bytes / 1024 ))KB"
    else
        echo "$(( bytes / 1048576 ))MB"
    fi
}

# Controleer of jq geïnstalleerd is
if ! command -v jq &> /dev/null; then
    echo -e "${RED}Error: jq is niet geïnstalleerd. Installeer dit eerst met: brew install jq${NC}"
    exit 1
fi

# Controleer of JSON bestand bestaat
if [[ ! -f "$JSON_FILE" ]]; then
    echo -e "${RED}Error: JSON bestand niet gevonden: $JSON_FILE${NC}"
    exit 1
fi

# Maak download directory aan als deze niet bestaat
mkdir -p "$DOWNLOAD_DIR"

# Initialiseer log files
echo "=== Download sessie gestart op $(date) ===" >> "$LOG_FILE"
> "$FAILED_FILE"

# Tel totaal aantal downloads
TOTAL=$(jq -r '.[].audio_url' "$JSON_FILE" | wc -l | tr -d ' ')
log "Totaal aantal te downloaden bestanden: $TOTAL"

# Variabelen voor statistieken
DOWNLOADED=0
SKIPPED=0
FAILED=0
UPDATED=0
CURRENT=0

# Array met user agents van populaire podcast players
USER_AGENTS=(
    "Spotify/8.8.0 iOS/16.0 (iPhone13,2)"
    "Overcast/2024.1 (iPhone; iOS 17.0; Scale/3.00)"
    "Apple Podcasts/2.0 (Macintosh; OS X 14.0)"
    "Pocket Casts/iOS 7.57.0"
    "Castro 2024.1/1234 iOS/17.0"
    "Google Podcasts/1.0"
)

# Functie om random user agent te selecteren
get_random_user_agent() {
    echo "${USER_AGENTS[$RANDOM % ${#USER_AGENTS[@]}]}"
}

# Functie om random timeout te genereren (tussen 3-8 seconden)
get_random_timeout() {
    echo $((3 + RANDOM % 6))
}

# Functie voor download met retry logic
download_with_retry() {
    local url=$1
    local output_file=$2
    local max_retries=3
    local retry_count=0
    
    while [[ $retry_count -lt $max_retries ]]; do
        local user_agent=$(get_random_user_agent)
        
        if curl -L \
            --fail \
            --silent \
            --show-error \
            --connect-timeout 30 \
            --max-time 300 \
            -H "User-Agent: $user_agent" \
            -H "Accept: audio/mpeg, audio/mp3, audio/*" \
            -H "Accept-Encoding: gzip, deflate, br" \
            -H "Accept-Language: nl-NL,nl;q=0.9,en;q=0.8" \
            -H "Cache-Control: no-cache" \
            -H "Referer: https://podcasts.apple.com/" \
            -H "X-Playback-Session-Id: $(uuidgen)" \
            --progress-bar \
            -o "$output_file" \
            "$url"; then
            return 0
        else
            retry_count=$((retry_count + 1))
            if [[ $retry_count -lt $max_retries ]]; then
                echo -e "${YELLOW}  Poging $retry_count/$max_retries mislukt. Opnieuw proberen in 10 seconden...${NC}"
                sleep 10
            fi
        fi
    done
    
    return 1
}

# Functie om remote file size op te halen
get_remote_size() {
    local url=$1
    local user_agent=$(get_random_user_agent)
    
    curl -sI -L \
        -H "User-Agent: $user_agent" \
        -H "Accept: audio/mpeg, audio/mp3, audio/*" \
        "$url" | \
        grep -i "content-length" | \
        tail -1 | \
        awk '{print $2}' | \
        tr -d '\r'
}

# Loop door alle URLs in JSON
while IFS= read -r line; do
    CURRENT=$((CURRENT + 1))
    
    # Extract URL en title
    url=$(echo "$line" | jq -r '.audio_url')
    title=$(echo "$line" | jq -r '.title')
    
    # Haal bestandsnaam uit URL
    filename=$(basename "$url")
    output_file="$DOWNLOAD_DIR/$filename"
    
    # Progress indicator
    echo -e "\n${BLUE}[$CURRENT/$TOTAL] Verwerken: $title${NC}"
    echo -e "${BLUE}Bestand: $filename${NC}"
    
    # Check of bestand al bestaat
    if [[ -f "$output_file" ]]; then
        local_size=$(stat -f%z "$output_file" 2>/dev/null || stat -c%s "$output_file" 2>/dev/null || echo "0")
        
        echo -e "${YELLOW}  Bestand bestaat al. Controleren grootte...${NC}"
        
        # Haal remote size op
        remote_size=$(get_remote_size "$url")
        
        if [[ -z "$remote_size" || "$remote_size" == "0" ]]; then
            echo -e "${YELLOW}  Waarschuwing: Kan remote grootte niet bepalen. Bestand overslaan.${NC}"
            log "[$title] Overgeslagen - kan remote grootte niet bepalen"
            SKIPPED=$((SKIPPED + 1))
            continue
        fi
        
        echo -e "  Lokaal: $(format_bytes $local_size), Remote: $(format_bytes $remote_size)"
        
        # Vergelijk groottes met 1% marge voor kleine verschillen
        size_diff=$((remote_size - local_size))
        if [[ $size_diff -gt $((remote_size / 100)) ]]; then
            echo -e "${YELLOW}  Lokaal bestand is kleiner. Opnieuw downloaden...${NC}"
            mv "$output_file" "$output_file.backup"
            
            if download_with_retry "$url" "$output_file"; then
                rm -f "$output_file.backup"
                echo -e "${GREEN}  ✓ Succesvol bijgewerkt!${NC}"
                log "[$title] Bijgewerkt - oude grootte: $(format_bytes $local_size), nieuwe grootte: $(format_bytes $remote_size)"
                UPDATED=$((UPDATED + 1))
            else
                mv "$output_file.backup" "$output_file"
                echo -e "${RED}  ✗ Update mislukt!${NC}"
                echo "$url|$title" >> "$FAILED_FILE"
                log "[$title] Update mislukt na 3 pogingen"
                FAILED=$((FAILED + 1))
            fi
        else
            echo -e "${GREEN}  ✓ Bestand is al compleet. Overslaan.${NC}"
            log "[$title] Overgeslagen - bestand al compleet"
            SKIPPED=$((SKIPPED + 1))
        fi
    else
        # Download nieuw bestand
        echo -e "${YELLOW}  Downloaden...${NC}"
        
        if download_with_retry "$url" "$output_file"; then
            final_size=$(stat -f%z "$output_file" 2>/dev/null || stat -c%s "$output_file" 2>/dev/null || echo "0")
            echo -e "${GREEN}  ✓ Succesvol gedownload! ($(format_bytes $final_size))${NC}"
            log "[$title] Gedownload - grootte: $(format_bytes $final_size)"
            DOWNLOADED=$((DOWNLOADED + 1))
        else
            echo -e "${RED}  ✗ Download mislukt na 3 pogingen!${NC}"
            echo "$url|$title" >> "$FAILED_FILE"
            log "[$title] Download mislukt na 3 pogingen"
            FAILED=$((FAILED + 1))
        fi
    fi
    
    # Random timeout tussen downloads (behalve bij laatste)
    if [[ $CURRENT -lt $TOTAL ]]; then
        timeout=$(get_random_timeout)
        echo -e "${BLUE}  Wachten $timeout seconden voor volgende download...${NC}"
        sleep $timeout
    fi
    
done < <(jq -c '.[]' "$JSON_FILE")

# Eindstatistieken
echo -e "\n${GREEN}=== Download Compleet ===${NC}"
echo -e "Nieuw gedownload: ${GREEN}$DOWNLOADED${NC}"
echo -e "Bijgewerkt:        ${YELLOW}$UPDATED${NC}"
echo -e "Overgeslagen:      ${BLUE}$SKIPPED${NC}"
echo -e "Mislukt:           ${RED}$FAILED${NC}"
echo -e "Totaal verwerkt:   $CURRENT"

log "Sessie beëindigd - Downloaded: $DOWNLOADED, Updated: $UPDATED, Skipped: $SKIPPED, Failed: $FAILED"

# Toon mislukte downloads indien aanwezig
if [[ $FAILED -gt 0 ]]; then
    echo -e "\n${RED}Mislukte downloads zijn opgeslagen in: $FAILED_FILE${NC}"
    echo -e "${YELLOW}Tip: Run het script opnieuw om mislukte downloads te herproberen.${NC}"
fi

# Check disk space na downloads
available_space=$(df -h "$DOWNLOAD_DIR" | awk 'NR==2 {print $4}')
echo -e "\n${BLUE}Beschikbare schijfruimte: $available_space${NC}"