Duplicut bash helper

duplicut_merger.sh

#!/usr/bin/env bash
set -euo pipefail

# Path to the duplicut binary. Shell aliases are never visible to scripts,
# so if `duplicut` is only a shell alias on your system, either:
#   1) run this script as:  DUPLICUT_BIN=/path/to/duplicut ./duplicut_merge.sh ...
#   2) or hardcode the path below.
DUPLICUT_BIN="${DUPLICUT_BIN:-duplicut}"

usage() {
    cat <<EOF
Usage: $(basename "$0") -f FILE [FILE ...] [-p] [-l LIMIT] -o OUTPUT

  -f, --files FILE [FILE ...]  Input files to merge (2+ files, globs work: -f *.txt)
  -p, --printable               Drop lines containing any non-printable-ASCII byte
  -l, --limit NUM                Drop lines longer than NUM characters
  -o, --output FILE              Output file for the merged, deduplicated result
  -h, --help                     Show this help

Examples:
  $(basename "$0") -f file1.txt file2.txt file3.txt -p -l 25 -o wordlist.merged
  $(basename "$0") -f wordlists/*.txt -l 20 -o merged.txt

Env:
  DUPLICUT_BIN   Path to the duplicut binary (default: "duplicut" via PATH).
EOF
}

FILES=()
PRINTABLE=false
LIMIT=""
OUTPUT=""

if [[ $# -eq 0 ]]; then
    usage
    exit 1
fi

while [[ $# -gt 0 ]]; do
    case "$1" in
        -f|--files)
            shift
            while [[ $# -gt 0 && "$1" != -* ]]; do
                FILES+=("$1")
                shift
            done
            ;;
        -p|--printable)
            PRINTABLE=true
            shift
            ;;
        -l|--limit)
            LIMIT="${2:-}"
            shift 2
            ;;
        -o|--output)
            OUTPUT="${2:-}"
            shift 2
            ;;
        -h|--help)
            usage
            exit 0
            ;;
        *)
            echo "Unknown option: $1" >&2
            usage
            exit 1
            ;;
    esac
done

if [[ ${#FILES[@]} -lt 1 ]]; then
    echo "Error: no input files given (-f)" >&2
    exit 1
fi

if [[ -z "$OUTPUT" ]]; then
    echo "Error: no output file given (-o)" >&2
    exit 1
fi

if [[ -n "$LIMIT" ]] && ! [[ "$LIMIT" =~ ^[0-9]+$ ]]; then
    echo "Error: -l/--limit must be a number, got: $LIMIT" >&2
    exit 1
fi

echo "[+] Merging ${#FILES[@]} file(s):"
for f in "${FILES[@]}"; do
    if [[ ! -f "$f" ]]; then
        echo "Error: file not found: $f" >&2
        exit 1
    fi
    echo "    - $f ($(wc -l < "$f") lines)"
done

TMP_MERGED="$(mktemp)"
trap 'rm -f "$TMP_MERGED"' EXIT

cat "${FILES[@]}" > "$TMP_MERGED"
echo "    Combined: $(wc -l < "$TMP_MERGED") lines (before dedup)"

echo
if command -v "$DUPLICUT_BIN" >/dev/null 2>&1; then
    echo "[+] Deduplicating with duplicut ($DUPLICUT_BIN)"

    CMD=("$DUPLICUT_BIN" "$TMP_MERGED" -o "$OUTPUT")
    [[ "$PRINTABLE" == "true" ]] && CMD+=(-p)
    [[ -n "$LIMIT" ]] && CMD+=(-l "$LIMIT")

    "${CMD[@]}"
else
    echo "[+] duplicut not found (DUPLICUT_BIN=$DUPLICUT_BIN), using sort/grep/awk fallback"

    if [[ "$PRINTABLE" == "true" ]]; then
        LC_ALL=C grep -av '[^ -~]' "$TMP_MERGED" > "${TMP_MERGED}.tmp" || true
        mv "${TMP_MERGED}.tmp" "$TMP_MERGED"
    fi

    if [[ -n "$LIMIT" ]]; then
        awk -v max="$LIMIT" 'length($0) <= max' "$TMP_MERGED" > "${TMP_MERGED}.tmp"
        mv "${TMP_MERGED}.tmp" "$TMP_MERGED"
    fi

    LC_ALL=C sort -u "$TMP_MERGED" > "$OUTPUT"
fi

echo
echo "[+] Done"
echo "    Output: $OUTPUT"
echo "    Lines: $(wc -l < "$OUTPUT")"