s3cp

s3cp

#!/usr/bin/env bash
# s3cp - aws s3 cp compatible wrapper around rclone
# Uses rclone's on-the-fly :s3,env_auth: backend (picks up AWS env/config)
#
# Usage: s3cp <src> <dst> [options]
#
# Behaves like `aws s3 cp`:
#   s3cp file.txt s3://bucket/prefix/          -> upload file, keep name
#   s3cp file.txt s3://bucket/prefix/new.txt   -> upload file, rename
#   s3cp s3://bucket/prefix/file.txt .         -> download file
#   s3cp s3://bucket/prefix/file.txt ./new.txt -> download & rename
#   s3cp dir  s3://bucket/prefix/              -> upload dir AS dir  (prefix/dir/…)
#   s3cp dir/ s3://bucket/prefix/              -> upload dir CONTENTS (prefix/…)
#   s3cp s3://bucket/prefix/ ./local/          -> download prefix contents
#   s3cp s3://bucket/key s3://bucket2/key      -> server-side copy
#
# Trailing-slash convention (same as cp / rsync / aws s3 cp):
#   source WITHOUT trailing /  -> the directory itself is placed inside dest
#   source WITH    trailing /  -> only the directory contents go into dest
#
# Directories are auto-detected: if the source is a local directory or an S3
# path ending with /, recursive mode is enabled automatically.
#
# Supported flags:
#   --recursive        copy directories recursively
#   --dryrun           show what would be copied without copying
#   --quiet            suppress output
#   --include PATTERN  include files matching pattern
#   --exclude PATTERN  exclude files matching pattern
#   --progress / -P    show real-time transfer stats
#   --                 stop parsing flags; remaining args passed to rclone

set -euo pipefail

# ─── defaults ────────────────────────────────────────────────────────────────
RECURSIVE=false
DRYRUN=false
QUIET=false
PROGRESS=false
INCLUDES=()
EXCLUDES=()
RCLONE_EXTRA=()

# ─── parse args ──────────────────────────────────────────────────────────────
POSITIONAL=()
while [[ $# -gt 0 ]]; do
    case "$1" in
        --recursive)
            RECURSIVE=true; shift ;;
        --dryrun|--dry-run)
            DRYRUN=true; shift ;;
        --quiet)
            QUIET=true; shift ;;
        --progress|-P)
            PROGRESS=true; shift ;;
        --include)
            INCLUDES+=("$2"); shift 2 ;;
        --exclude)
            EXCLUDES+=("$2"); shift 2 ;;
        --)
            shift; RCLONE_EXTRA+=("$@"); break ;;
        -*)
            RCLONE_EXTRA+=("$1"); shift ;;
        *)
            POSITIONAL+=("$1"); shift ;;
    esac
done

if [[ ${#POSITIONAL[@]} -lt 2 ]]; then
    echo "Usage: s3cp <source> <destination> [--recursive] [--dryrun] [--quiet] [--exclude PATTERN] [--include PATTERN]" >&2
    exit 1
fi

SRC="${POSITIONAL[0]}"
DST="${POSITIONAL[1]}"

# ─── resolve AWS region ──────────────────────────────────────────────────────
# Mirrors the AWS SDK lookup order: env vars -> profile in ~/.aws/config
resolve_aws_region() {
    # 1. Explicit env vars
    if [[ -n "${AWS_REGION:-}" ]]; then
        echo "$AWS_REGION"; return
    fi
    if [[ -n "${AWS_DEFAULT_REGION:-}" ]]; then
        echo "$AWS_DEFAULT_REGION"; return
    fi
    # 2. Active profile in ~/.aws/config
    local profile="${AWS_PROFILE:-default}"
    local config="${AWS_CONFIG_FILE:-$HOME/.aws/config}"
    if [[ -f "$config" ]]; then
        local region
        region=$(awk -v profile="$profile" '
            BEGIN { found=0 }
            /^\[/ {
                found=0
                gsub(/^\[[ \t]*|[ \t]*\]$/, "")
                # match [profile X] or [X] (for default)
                sub(/^profile[ \t]+/, "")
                if ($0 == profile) found=1
            }
            found && /^[ \t]*region[ \t]*=/ {
                sub(/^[^=]+=[ \t]*/, "")
                print; exit
            }
        ' "$config")
        if [[ -n "$region" ]]; then
            echo "$region"; return
        fi
    fi
    # 3. Fallback
    echo "us-east-1"
}

S3_REGION="$(resolve_aws_region)"

# ─── helpers ─────────────────────────────────────────────────────────────────

# Build the on-the-fly rclone S3 backend prefix:
#   :s3,env_auth,region=X,no_check_bucket:
# - env_auth: pick up creds from env / ~/.aws / instance metadata
# - region: avoids "IllegalLocationConstraintException" on bucket access
# - no_check_bucket: prevents rclone from trying to CreateBucket
S3_BACKEND=":s3,env_auth,region=${S3_REGION},no_check_bucket:"

# Convert s3://bucket/path to <S3_BACKEND>bucket/path
s3_to_rclone() {
    local p="$1"
    if [[ "$p" =~ ^s3://(.*)$ ]]; then
        echo "${S3_BACKEND}${BASH_REMATCH[1]}"
    else
        echo "$p"
    fi
}

# Convert rclone backend notation back to s3:// (for display)
rclone_to_s3() {
    local p="$1"
    if [[ "$p" == "${S3_BACKEND}"* ]]; then
        echo "s3://${p#"${S3_BACKEND}"}"
    else
        echo "$p"
    fi
}

is_s3() {
    [[ "$1" =~ ^s3:// ]]
}

# Check if a local path is a file (exists and is a regular file)
is_local_file() {
    [[ -f "$1" ]]
}

# Check if a local path is a directory (exists and is a directory)
is_local_dir() {
    [[ -d "$1" ]]
}

# Check if a path looks like it refers to a file (has an extension in the last
# component OR does not end with /).  This is a heuristic for remote paths
# where we can't stat.
looks_like_file() {
    local p="$1"
    # Ends with / -> definitely directory-ish
    [[ "$p" == */ ]] && return 1
    # Get the last component after the final /
    local base="${p##*/}"
    # Contains a dot -> likely a file
    [[ "$base" == *.* ]] && return 0
    # No dot, no trailing slash -> ambiguous, treat as directory (safe default
    # matching aws s3 cp behavior where a bare prefix is a key prefix)
    return 1
}

# Extract the filename from a path (local or s3://)
basename_of() {
    local p="${1%/}"
    echo "${p##*/}"
}

ends_with_slash() {
    [[ "$1" == */ ]]
}

# ─── translate src/dst ───────────────────────────────────────────────────────
R_SRC="$(s3_to_rclone "$SRC")"
R_DST="$(s3_to_rclone "$DST")"

# ─── auto-detect directories ─────────────────────────────────────────────────
# If source is a local directory or an S3 prefix ending with /, auto-enable
# recursive mode so the user doesn't have to remember --recursive.
if ! $RECURSIVE; then
    if ! is_s3 "$SRC"; then
        # Local source: check filesystem
        if is_local_dir "$SRC"; then
            RECURSIVE=true
        fi
    else
        # S3 source: trailing slash is the only reliable hint for a prefix
        if ends_with_slash "$SRC"; then
            RECURSIVE=true
        fi
    fi
fi

# ─── determine operation ────────────────────────────────────────────────────
# 1. recursive (explicit or auto-detected): `rclone copy` (dir to dir)
# 2. single file:
#    a. dst ends with / or is existing dir -> copyto src dst/basename(src)
#    b. otherwise                          -> copyto src dst

RCLONE_CMD=""
FINAL_SRC="$R_SRC"
FINAL_DST="$R_DST"

if $RECURSIVE; then
    # ── directory / prefix copy ──────────────────────────────────────────
    RCLONE_CMD="copy"
    # rclone copy always copies *contents* of the source directory.
    #
    # Trailing-slash convention (matches cp / rsync / aws s3 cp):
    #   src WITHOUT trailing /  -> copy the dir itself into dst
    #                              e.g. .github -> dst/.github/…
    #   src WITH trailing /     -> copy only the contents into dst
    #                              e.g. .github/ -> dst/…
    #
    # Since rclone always copies contents, when there is NO trailing slash
    # we append the source directory basename to the destination so the
    # directory itself ends up inside the target.
    if ! ends_with_slash "$SRC"; then
        src_base="$(basename_of "$SRC")"
        R_DST="${R_DST%/}/${src_base}"
    fi
    FINAL_SRC="${R_SRC%/}"
    FINAL_DST="${R_DST%/}"
else
    # ── single-file copy ─────────────────────────────────────────────────
    RCLONE_CMD="copyto"

    if ! is_s3 "$SRC" && ! is_local_file "$SRC"; then
        # Source doesn't exist locally - let rclone report the error
        :
    fi

    # Now figure out the destination
    if ! is_s3 "$DST"; then
        # ── local destination ────────────────────────────────────────────
        if ends_with_slash "$DST" || is_local_dir "$DST"; then
            # Dest is a directory: append the source filename
            local_dir="${DST%/}"
            src_base="$(basename_of "$SRC")"
            FINAL_DST="${local_dir}/${src_base}"
            # Make sure the directory exists
            mkdir -p "$local_dir"
        fi
        # else: dest is a file path, use as-is (already set)
    else
        # ── S3 destination ───────────────────────────────────────────────
        if ends_with_slash "$DST"; then
            # s3://bucket/prefix/ -> append source basename
            src_base="$(basename_of "$SRC")"
            # R_DST already has the trailing slash converted
            FINAL_DST="${R_DST}${src_base}"
        elif ! looks_like_file "$DST"; then
            # Bare prefix without trailing slash and no extension:
            # aws s3 cp treats this as a key (not a prefix), so copy as-is
            :
        fi
        # else: has extension or specific key - use as-is
    fi

    # Re-convert FINAL_DST if it was rebuilt from DST (not R_DST)
    FINAL_DST="$(s3_to_rclone "$(rclone_to_s3 "$FINAL_DST")" 2>/dev/null || echo "$FINAL_DST")"
fi

# ─── build rclone command ───────────────────────────────────────────────────
CMD=(rclone "$RCLONE_CMD")

if $DRYRUN;   then CMD+=(--dry-run); fi
if $QUIET;    then CMD+=(-q); fi
if $PROGRESS; then CMD+=(-P); fi

for pat in "${INCLUDES[@]}"; do
    CMD+=(--include "$pat")
done
for pat in "${EXCLUDES[@]}"; do
    CMD+=(--exclude "$pat")
done

CMD+=("${RCLONE_EXTRA[@]}")
CMD+=("$FINAL_SRC" "$FINAL_DST")

# ─── show what we're doing (unless quiet) ────────────────────────────────────
if ! $QUIET; then
    # Pretty-print in aws-style
    action="copy"
    if is_s3 "$SRC" && ! is_s3 "$DST"; then
        action="download"
    elif ! is_s3 "$SRC" && is_s3 "$DST"; then
        action="upload"
    fi
    # Convert rclone notation back to s3:// for display
    display_src="$(rclone_to_s3 "$FINAL_SRC")"
    display_dst="$(rclone_to_s3 "$FINAL_DST")"
    echo "${action}: ${display_src} to ${display_dst}"
fi

# ─── run ─────────────────────────────────────────────────────────────────────
if $DRYRUN && ! $QUIET; then
    echo "rclone command: ${CMD[*]}"
fi

exec "${CMD[@]}"