Batch downloading scientific articles from sci-hub via DOI

download_papers.sh

#!/usr/bin/env bash
# =============================================================================
# download_papers.sh — Download scientific articles from Sci-Hub via DOI list
#
# Usage:
#   chmod +x download_papers.sh
#   ./download_papers.sh dois.txt
#
# The input file should contain one DOI per line, e.g.:
#   10.1038/nature12373
#   10.1016/j.cell.2015.05.002
#   https://doi.org/10.1126/science.abc1234   ← full URLs are also handled
#
# Downloaded PDFs are saved to ./papers/ with sanitised filenames.
# A log file (download.log) records success/failure for each DOI.
# =============================================================================

set -euo pipefail

# ── Configuration ────────────────────────────────────────────────────────────
SCIHUB_BASE="https://www.sci-hub.in"
OUTPUT_DIR="./papers"
LOG_FILE="./download.log"
DELAY=3          # seconds to wait between requests (be polite)
MAX_RETRIES=3    # number of retry attempts per DOI
USER_AGENT="Mozilla/5.0 (X11; Linux x86_64; rv:124.0) Gecko/20100101 Firefox/124.0"

# ── Colours (disabled when not a TTY) ────────────────────────────────────────
if [ -t 1 ]; then
  GREEN="\033[0;32m"; YELLOW="\033[1;33m"; RED="\033[0;31m"; RESET="\033[0m"
else
  GREEN=""; YELLOW=""; RED=""; RESET=""
fi

# ── Helpers ──────────────────────────────────────────────────────────────────
log()  { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }
info() { echo -e "${GREEN}[INFO]${RESET}  $*"; }
warn() { echo -e "${YELLOW}[WARN]${RESET}  $*"; }
err()  { echo -e "${RED}[ERROR]${RESET} $*" >&2; }

check_deps() {
  local missing=()
  for cmd in curl pup grep sed; do
    command -v "$cmd" &>/dev/null || missing+=("$cmd")
  done
  # pup is optional; fall back to grep-based parsing if absent
  if ! command -v pup &>/dev/null; then
    warn "'pup' not found — falling back to grep/sed for HTML parsing (less robust)."
  fi
  if [[ ${#missing[@]} -gt 0 ]]; then
    # Remove pup from hard-required list
    local hard_missing=()
    for m in "${missing[@]}"; do [[ "$m" != "pup" ]] && hard_missing+=("$m"); done
    if [[ ${#hard_missing[@]} -gt 0 ]]; then
      err "Missing required tools: ${hard_missing[*]}"
      err "Install them and re-run (e.g. sudo apt-get install curl)"
      exit 1
    fi
  fi
}

# Strip leading/trailing whitespace and normalise DOI
normalise_doi() {
  local raw="$1"
  # Remove full URL prefix if present
  raw="${raw#https://doi.org/}"
  raw="${raw#http://doi.org/}"
  raw="${raw#https://dx.doi.org/}"
  raw="${raw#http://dx.doi.org/}"
  # Trim whitespace
  raw="$(echo "$raw" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')"
  echo "$raw"
}

# Turn a DOI into a safe filename (replace / and special chars)
doi_to_filename() {
  echo "$1" | sed 's|/|_|g; s|[^A-Za-z0-9._-]|_|g'
}

# Extract the PDF URL from a Sci-Hub HTML page
extract_pdf_url() {
  local html="$1"
  local pdf_url=""

  if command -v pup &>/dev/null; then
    # Use pup for reliable CSS-selector parsing
    pdf_url=$(echo "$html" | pup '#pdf attr{src}' 2>/dev/null | head -1)
    # Some pages embed it inside an <iframe>
    if [[ -z "$pdf_url" ]]; then
      pdf_url=$(echo "$html" | pup 'iframe attr{src}' 2>/dev/null | grep -i '\.pdf' | head -1)
    fi
  fi

  # Fallback: grep for common PDF embed patterns
  if [[ -z "$pdf_url" ]]; then
    pdf_url=$(echo "$html" | grep -oP '(?<=src=")[^"]*\.pdf[^"]*' | head -1 || true)
  fi
  if [[ -z "$pdf_url" ]]; then
    pdf_url=$(echo "$html" | grep -oP "(?<=src=')[^']*\.pdf[^']*" | head -1 || true)
  fi
  # downloads API path (no .pdf extension)
  if [[ -z "$pdf_url" ]]; then
    pdf_url=$(echo "$html" | grep -oP '(?<=location\.href=\\?")[^"\\]+' | grep -i 'download\|pdf' | head -1 || true)
  fi

  echo "$pdf_url"
}

# Download a single DOI; returns 0 on success, 1 on failure
download_doi() {
  local doi="$1"
  local safe_name
  safe_name=$(doi_to_filename "$doi")
  local out_path="${OUTPUT_DIR}/${safe_name}.pdf"

  if [[ -f "$out_path" ]]; then
    warn "Already downloaded: $doi — skipping."
    log "SKIP $doi"
    return 0
  fi

  local scihub_url="${SCIHUB_BASE}/${doi}"
  info "Fetching Sci-Hub page: $scihub_url"

  # Step 1 — fetch the Sci-Hub landing page
  local html
  html=$(curl --silent --max-time 30 --retry 2 \
    -A "$USER_AGENT" \
    -L "$scihub_url" 2>/dev/null) || { err "Could not reach $scihub_url"; return 1; }

  # Step 2 — extract PDF link
  local pdf_url
  pdf_url=$(extract_pdf_url "$html")

  if [[ -z "$pdf_url" ]]; then
    err "Could not find PDF link for DOI: $doi"
    log "FAIL (no PDF link) $doi"
    return 1
  fi

  # Prepend scheme+host when URL is protocol-relative or path-only
  if [[ "$pdf_url" == //* ]]; then
    pdf_url="https:${pdf_url}"
  elif [[ "$pdf_url" == /* ]]; then
    pdf_url="${SCIHUB_BASE}${pdf_url}"
  fi

  info "Downloading PDF: $pdf_url"

  # Step 3 — download the PDF
  local http_code
  http_code=$(curl --silent --max-time 60 \
    -A "$USER_AGENT" \
    -L "$pdf_url" \
    -o "$out_path" \
    --write-out "%{http_code}" 2>/dev/null) || { err "curl failed for $pdf_url"; return 1; }

  if [[ "$http_code" -ne 200 ]]; then
    err "HTTP $http_code for $pdf_url"
    rm -f "$out_path"
    log "FAIL (HTTP $http_code) $doi"
    return 1
  fi

  # Sanity-check: is the downloaded file actually a PDF?
  local magic
  magic=$(head -c 4 "$out_path" 2>/dev/null || true)
  if [[ "$magic" != "%PDF" ]]; then
    err "Downloaded file is not a PDF for DOI: $doi"
    rm -f "$out_path"
    log "FAIL (not a PDF) $doi"
    return 1
  fi

  local size
  size=$(du -sh "$out_path" | cut -f1)
  info "Saved: $out_path ($size)"
  log "OK $doi -> $out_path"
  return 0
}

# ── Main ─────────────────────────────────────────────────────────────────────
main() {
  if [[ $# -lt 1 ]]; then
    echo "Usage: $0 <doi_list_file>"
    echo "       Each line of the file should contain one DOI or DOI URL."
    exit 1
  fi

  local doi_file="$1"
  if [[ ! -f "$doi_file" ]]; then
    err "File not found: $doi_file"
    exit 1
  fi

  check_deps

  mkdir -p "$OUTPUT_DIR"
  : > "$LOG_FILE"   # truncate / create log

  local total=0 success=0 failed=0
  local failed_dois=()

  while IFS= read -r raw_line || [[ -n "$raw_line" ]]; do
    # Skip blank lines and comments
    [[ -z "$raw_line" || "$raw_line" == \#* ]] && continue

    local doi
    doi=$(normalise_doi "$raw_line")
    [[ -z "$doi" ]] && continue

    ((total++)) || true
    info "── DOI $total: $doi"

    local attempt=0 ok=false
    while [[ $attempt -lt $MAX_RETRIES ]]; do
      ((attempt++)) || true
      if download_doi "$doi"; then
        ok=true
        break
      else
        warn "Attempt $attempt/$MAX_RETRIES failed for $doi"
        [[ $attempt -lt $MAX_RETRIES ]] && sleep "$DELAY"
      fi
    done

    if $ok; then
      ((success++)) || true
    else
      ((failed++)) || true
      failed_dois+=("$doi")
    fi

    # Polite delay between articles
    sleep "$DELAY"

  done < "$doi_file"

  echo ""
  echo "═══════════════════════════════════════"
  echo " Summary"
  echo "═══════════════════════════════════════"
  echo " Total processed : $total"
  echo -e " ${GREEN}Succeeded${RESET}        : $success"
  echo -e " ${RED}Failed${RESET}           : $failed"
  echo " Output directory: $OUTPUT_DIR"
  echo " Log file        : $LOG_FILE"

  if [[ ${#failed_dois[@]} -gt 0 ]]; then
    echo ""
    warn "Failed DOIs:"
    for d in "${failed_dois[@]}"; do echo "  - $d"; done
  fi
  echo "═══════════════════════════════════════"
}

main "$@"