dedup-test

#!/usr/bin/env bash
# shellcheck disable=SC2016

set -euo pipefail

# for system in */; do
#     system="${system%/}"
#     echo >&2 "$system.."
#     {
#         echo "$system"
#         find ../Devices -mindepth 2 -maxdepth 2 -name "$system" 2>/dev/null
#     } |
#         nlxargs fd -t f '' |
#         { grep -Ev 'Widescreen|\.scummvm$|\.dreamm$|\.pcm|\.lnk|\.sh|\.bat|\.desktop' || :; } |
#             xargs-paste -n 1 bash -c 'basename "$1" | sed -e "s/\.[^.]*$//"' - |
#         kdedup -r |
#         jdupes-remove-nondupes |
#         ifne fdupes-xargs ./link-dupes-basename
# done

for system in */; do
    system="${system%/}"
    echo >&2 "$system.."
    {
        echo "$system"
        find ../Devices -mindepth 2 -maxdepth 2 -name "$system" 2>/dev/null
    } |
        nlxargs fd -t f '' |
        { grep -Ev 'Widescreen|\.scummvm$|\.dreamm$|\.pcm|\.lnk|\.sh|\.bat|\.desktop' || :; } |
            xargs-paste -n 1 bash -c 'basename "$1" | sed -e "s/\.[^.]*$//; s/([^)]*)//g; s/\[[^]]*\]//g; s/ *$//; s/^ *//;"' - |
        kdedup -r |
        jdupes-remove-nondupes |
        ifne fdupes-xargs ./link-dupes-nosuffix
done
# ./fdupes-xargs bash -c 'if ! same-file "$@"; then; if echo "$@" | grep -q /Best; then nonbest=; for f; do; if ! echo "$f" | grep -q /Best; then nonbest="$f"; fi; done; if [ -n "$nonbest" ]; then echo nonbest: $nonbest; for i in "$@"; do; if [ "$i" != "$nonbest" ]; then; ln -fv "$nonbest" "$i"; fi; done; fi; fi; fi' -

fdupes-xargs

#!/usr/bin/env bash

set -euo pipefail

jdupes-oneline |
    while read -r line; do
        IFS=$'\t' read -d '' -r -a files < <(echo "$line" | sed -e 's/\\ / /g' | tr -d '\n') || :
        for file in "${files[@]}"; do
            echo "$file"
        done |
            tr '\n' '\0' |
            xargs -0 "$@"
    done

jdupes-multiline

#!/usr/bin/env python3
# Given fdupes oneline output, output grouped multiline output like jdupes

import os
import sys


def main():
    first = True
    for line in sys.stdin:
        entries = line.strip('\r\n').split('\t')
        if first:
            first = False
        else:
            sys.stdout.write('\n')
        sys.stdout.writelines(entry.replace('\\ ', ' ') + '\n' for entry in entries)


if __name__ == '__main__':
    try:
        main()
        sys.stdout.flush()
    except BrokenPipeError:
        # Python flushes standard streams on exit; redirect remaining output
        # to devnull to avoid another BrokenPipeError at shutdown
        devnull = os.open(os.devnull, os.O_WRONLY)
        os.dup2(devnull, sys.stdout.fileno())
        sys.exit(1)  # Python exits with error code 1 on EPIPE

jdupes-oneline

#!/usr/bin/env python3
"""Adjust jdupes output to match fdupes oneline output."""

import os
import sys


def main():
    """Adjust jdupes output to match fdupes oneline output."""
    block = []
    for line in sys.stdin:
        line = line.rstrip('\r\n')
        if not line:
            if block:
                print('\t'.join(block))
            block.clear()
        else:
            block.append(line.replace(' ', '\\ '))

    if block:
        print('\t'.join(block))


if __name__ == '__main__':
    try:
        main()
        sys.stdout.flush()
    except BrokenPipeError:
        # Python flushes standard streams on exit; redirect remaining output
        # to devnull to avoid another BrokenPipeError at shutdown
        devnull = os.open(os.devnull, os.O_WRONLY)
        os.dup2(devnull, sys.stdout.fileno())
        sys.exit(1)  # Python exits with error code 1 on EPIPE

jdupes-remove-missing

#!/usr/bin/env python3
# Given multiline dupe output, remove any missing files

import os
import sys


def main():
    for line in sys.stdin:
        line = line.rstrip('\r\n')
        if not line or os.path.exists(line):
            print(line)


if __name__ == '__main__':
    try:
        main()
        sys.stdout.flush()
    except BrokenPipeError:
        # Python flushes standard streams on exit; redirect remaining output
        # to devnull to avoid another BrokenPipeError at shutdown
        devnull = os.open(os.devnull, os.O_WRONLY)
        os.dup2(devnull, sys.stdout.fileno())
        sys.exit(1)  # Python exits with error code 1 on EPIPE

jdupes-remove-nondupes

#!/usr/bin/env python3
# Given multiline dupe output, remove any with a single entry

import os
import sys


def print_block(block, first):
    if len(block) > 1:
        if not first:
            sys.stdout.write('\n')
        sys.stdout.writelines(block)
        return True


def main():
    first, block = True, []
    for line in sys.stdin:
        if not line.rstrip('\r\n'):
            if print_block(block, first):
                first = False
            block.clear()
        else:
            block.append(line)

    if block:
        print_block(block, first)


if __name__ == '__main__':
    try:
        main()
        sys.stdout.flush()
    except BrokenPipeError:
        # Python flushes standard streams on exit; redirect remaining output
        # to devnull to avoid another BrokenPipeError at shutdown
        devnull = os.open(os.devnull, os.O_WRONLY)
        os.dup2(devnull, sys.stdout.fileno())
        sys.exit(1)  # Python exits with error code 1 on EPIPE

kdedup

#!/bin/sh
# - Accept file paths on stdin
# - Accept the identifier command via command-line argument
# - Output files grouped by the identifier
#
# NOTE: Currently assumes the command will output '<id><TAB
# CHARACTER><filename>' lines

PATH=$(dirname "$0"):$PATH
TAB=$(printf '\t')

usage() {
    cat <<END >&2
${0##*/} [options..] [CMD [ARGS..]]

Options:
  -a   Also show non-duplicates
  -r   Reverse the expected order from <id> <fn> to <fn> <id>.
  -h   Show usage
END
    exit 2
}

all=0
reversed_order=0
while getopts arh opt; do
    case "$opt" in
        a)
            all=1
            ;;
        r)
            reversed_order=1
            ;;
        \? | h)
            usage
            ;;
    esac
done
shift $((OPTIND - 1))

if echo "$*" | grep -q '{}'; then
    set -- -I"{}" "$@"
fi

remove_nondupes() {
    jdupes-oneline |
        sed -e "s/$TAB*$//" |
        grep "$TAB" |
        jdupes-multiline
}

if [ $# -gt 0 ]; then
    tr '\n' '\0' \
        | xargs -0 "$@"
else
    cat
fi |
    if [ $reversed_order -eq 1 ]; then
        grep -v "$TAB *$" |
        sort -t"$TAB" -k2 |
        group-by-column 2 "$TAB" |
        cut -d"$TAB" -f1
    else
        grep -v "^ *$TAB" |
        sort -t"$TAB" -k1 |
        group-by-column 1 "$TAB" |
        cut -d"$TAB" -f2-
    fi |
    if [ $all -eq 1 ]; then
        cat
    else
        remove_nondupes
    fi

link-dupes-basename

#!/usr/bin/env bash

set -euo pipefail

# 0. Exit if nothing to be done
if [ $# -eq 0 ]; then
    echo >&2 "Nothing to do"
    exit 0
fi

# 1. If the games are already hard linked, exit
if same-file "$@"; then
    exit 0
fi

# 2. If any game isn't in the "Best" folders, prefer that
preferred=
for game; do
    case "$game" in
        */Best*)
            ;;
        *)
            preferred="$game"
            ;;
    esac
done

if [ -z "$preferred" ]; then
    # 3. If any game is in 'Best', prefer that over the other 'Best' folders
    for game; do
        case "$game" in
            */Best/*)
                preferred="$game"
                ;;
            *)
                ;;
        esac
    done
else
    echo >&2 "Non-best: $preferred"
fi

if [ -z "$preferred" ]; then
    preferred="$1"
    echo >&2 "No preferred non-best or 'Best' game identified, assuming the first: $preferred"
    shift
else
    echo >&2 "Preferred 'Best': $preferred"
fi

for game; do
    if [ "$game" != "$preferred" ]; then
        gamedir=$(dirname "$game")
        mv "$game" "$game.old"
        ln -fv "$preferred" "$gamedir/" ||
            cp -fv "$preferred" "$gamedir/" ||
            mv "$game.old" "$game"
    fi
done

link-dupes-nosuffix

#!/usr/bin/env bash

set -euo pipefail

# 0. Exit if nothing to be done
if [ $# -eq 0 ]; then
    exit 0
fi

# 1. If the games are already hard linked, exit
if same-file "$@"; then
    exit 0
fi

# If they share the same basename, skip, as we already did that
differs=0
base=$(basename "${1%.*}")
for game; do
    if [ "$(basename "${game%.*}")" != "$base" ]; then
        differs=1
    fi
done
if [ $differs -eq 0 ]; then
    exit 0
fi

# 2. If any game isn't in the "Best" folders, prefer that
preferred=
for game; do
    case "$game" in
        */Best*)
            ;;
        *)
            preferred="$game"
            ;;
    esac
done

if [ -z "$preferred" ]; then
    # If there's a game without a suffix, and one with, prefer the latter
    nosuffix=
    suffixed=
    for game; do
        case "$game" in
            *\(*)
                suffixed="$game"
                ;;
            *\[*)
                suffixed="$game"
                ;;
            *)
                nosuffix="$game"
                ;;
        esac
        if [ -n "$nosuffix" ] && [ -n "$suffixed" ]; then
            preferred="$suffixed"
        fi
    done
fi

if [ -z "$preferred" ]; then
    # 3. If any game is in 'Best', prefer that over the other 'Best' folders
    for game; do
        case "$game" in
            */Best/*)
                preferred="$game"
                ;;
            *)
                ;;
        esac
    done
fi

if [ -z "$preferred" ]; then
    preferred="$1"
    echo >&2 "No preferred game identified, exiting"
    exit 0
else
    echo >&2 "Preferred: $preferred"
fi

for game; do
    if [ "$game" != "$preferred" ]; then
        gamedir=$(dirname "$game")
        echo >&2 "$gamedir/$(basename "$preferred") -> $preferred"
        continue
        mv "$game" "$game.old"
        ln -fv "$preferred" "$gamedir/" ||
            cp -fv "$preferred" "$gamedir/" ||
            mv "$game.old" "$game"
        rm -f "$game.old"
    fi
done