Remote sync files with various method for various protocol

-remote_sync_files.md

Remote sync/copy files with various method for various protocol

copy_with_tool_rclone_protocol_http_1fichier.sh

#!/usr/bin/env bash
# https://gist.github.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801
# 1fichier shared folder : https://1fichier.com/dir/folder_id
# NOTE : this method do not sync but use copy method of rclone. So files not existing in source will not be deleted from the destination folder

# Do this at least once to add a rclone source named "foo" of type "fichier" for the given shared_folder
# (cat <<'EOL'
# [foo]
# type = fichier
# shared_folder = folder_id
# api_key = xxxxxxxxxxxxxx
# EOL
# )>> "${HOME}/.config/rclone/rclone.conf"

CURRENT_FILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORKSPACE="${CURRENT_FILE_DIR}"
rm -f "${WORKSPACE}"/*.log


process_rclone_copy() {
   # generate a list of files to process
   rclone lsl --human-readable "${URI}" >"${WORKSPACE}/list-${NICK_NAME}.log"
    
  case $MODE in
    test)
      # test run in dry-run mode
      echo "** Test mode activated, only a dry-run will be made **"
      rclone copy --dry-run -vv --human-readable --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"
    ;;
    
    *)
      # launch copy
      # NOTE : this is a copy, not a sync. Files are not deleted from destination if not exist in source
      rclone copy -vv --human-readable --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"
    ;;
	esac
}
usage() {
	echo "Usage : $0 [test]"
}
if [ "$1" = "test" ]; then
	echo "** Test mode activated, only a dry-run will be made **"
	MODE="test"
fi
usage

check() {
	[ ! -f "${HOME}/.config/rclone/rclone.conf" ] && { echo "Missing rclone conf ${HOME}/.config/rclone/rclone.conf" > "${WORKSPACE}/log-${NICK_NAME}.log"; exit 1; }
	if ! command -v rclone >/dev/null 2>&1; then
		local msg="ERROR: Missing rclone binary (PATH=$PATH)"
		echo "$msg" | tee -a "${WORKSPACE}/log-${NICK_NAME}.log" >&2
		exit 1
	fi
}



NICK_NAME="foo"
URI="foo:"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content"
check
process_rclone_copy

sync_with_tool_rclone_protocol_http_internetarchive.sh

#!/usr/bin/env bash
# https://gist.github.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801
# Sync file from 2 internet archive url into one folder
# Synched files : https://archive.org/details/foo1
#		          https://archive.org/details/foo2

# Do this at least once to add a rclone source named "internetarchive" of type "internetarchive"
# (cat <<'EOL'
# [internetarchive]
# type = internetarchive
# EOL
# )>> "${HOME}/.config/rclone/rclone.conf"

CURRENT_FILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORKSPACE="${CURRENT_FILE_DIR}"
rm -f "${WORKSPACE}"/*.log



process_rclone_sync() {
  # generate a list of files to process
  rclone lsl "${URI}" --human-readable --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata" >"${WORKSPACE}/list-${NICK_NAME}.log"

  case $MODE in
    test)
      # test run in dry-run mode
      echo "** Test mode activated, only a dry-run will be made **"
      rclone sync --dry-run -vv --human-readable --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata" --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${WORKSPACE}/content/${NICK_NAME}"
    ;;

    *)
      # launch sync
      rclone sync -vv --human-readable --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata" --log-file="${WORKSPACE}/sync-${NICK_NAME}.log" "${URI}" "${WORKSPACE}/content/${NICK_NAME}"
    ;;
  esac
}
usage() {
	echo "Usage : $0 [test]"
}
if [ "$1" = "test" ]; then
	echo "** Test mode activated, only a dry-run will be made **"
	MODE="test"
fi
usage


check() {
	[ ! -f "${HOME}/.config/rclone/rclone.conf" ] && { echo "Missing rclone conf ${HOME}/.config/rclone/rclone.conf" > "${WORKSPACE}/log-${NICK_NAME}.log"; exit 1; }
	if ! command -v rclone >/dev/null 2>&1; then
		local msg="ERROR: Missing rclone binary (PATH=$PATH)"
		echo "$msg" | tee -a "${WORKSPACE}/log-${NICK_NAME}.log" >&2
		exit 1
	fi
}


# PART1
NICK_NAME="foo-part1"
# NOTE foo1 is the subpath of the internet archive url
URI="internetarchive:foo1"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content/${NICK_NAME}"

check
process_rclone_sync


# PART2
NICK_NAME="foo-part2"
# NOTE foo2 is the subpath of the internetarchive url
URI="internetarchive:foo2"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content/${NICK_NAME}"

check
process_rclone_sync

sync_with_tool_rclone_with_filter.sh

#!/usr/bin/env bash
# https://gist.github.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801#file-sync_with_tool_rclone_with_filter-sh
# define variables before sourcing this script and define RCLONE_BACKEND to define source to sync
#   NICK_NAME="project-nick-name"
#   RCLONE_METHOD="sync" # sync or copy
#   RCLONE_BACKEND=()
#   RCLONE_BACKEND+=( :http: ) # rclone backend
#   RCLONE_BACKEND+=( --http-url="https://myrient.erista.me/files/No-Intro")
#   PRIORITY="include" # include or exclude
#   export WORKSPACE="${CURRENT_FILE_DIR}/workspace-${NICK_NAME}"
__CURRENT_FILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

[ -z "${NICK_NAME:-}" ] && ( echo "Missing NICK_NAME environment variable as task name" && exit 1 )
[ -z "${RCLONE_METHOD:-}" ] && ( echo "Missing RCLONE_METHOD environment variable : sync OR copy" && exit 1 )
[ -z "${RCLONE_BACKEND:-}" ] && ( echo "Missing RCLONE_BACKEND environment variable to define rclone source" && exit 1 )
[ -z "${PRIORITY:-}" ] && ( echo "Missing PRIORITY environment variable : include OR exclude" && exit 1 )
[ -z "${TARGET:-}" ] && ( echo "Missing TARGET environment variable for target path" && exit 1 )

# prepare workspace
[ -z "${WORKSPACE:-}" ] && export WORKSPACE="${__CURRENT_FILE_DIR}/workspace-${NICK_NAME}"
mkdir -p "${WORKSPACE}"
rm -f "${WORKSPACE}"/*.log

FILTERS_FILE="${WORKSPACE}/filter.txt"

# NOTE rclone : cannot use together --include-from and --exclude-from because the order they are parsed in is indeterminate. Use --filter instead is recommended
process_rclone_sync_with_filter() {
  check
  build_filter_file
  
  

  LIST_FLAGS=()
  if [ -n "${FILTERS_FILE:-}" ]; then
    LIST_FLAGS+=( --filter-from="${FILTERS_FILE}" )
    cat "${FILTERS_FILE:-}"
  fi

  [ "${RCLONE_METHOD}" = "sync" ] && LIST_FLAGS+=( --delete-excluded )
  
  [ "${MODE}" = "test" ] && echo "** Test mode activated, only a dry-run will be made **"

  echo "Generating a list of files to process"
  # rclone lsl "${RCLONE_BACKEND[@]}" "${LIST_FLAGS[@]}" --human-readable > "${WORKSPACE}/list-${NICK_NAME}.log"
  # NOTE : rclone old version does not support --human-readable, so we convert the size field to human readable format with numfmt
  rclone lsl "${RCLONE_BACKEND[@]}" "${LIST_FLAGS[@]}" \
    | awk '{size=$1; $1=""; sub(/^ /,""); print size "\t" $0}' \
    | numfmt --field=1 --to=iec \
    | awk -F '\t' '{printf "%8s %s\n", $1, $2}' > "${WORKSPACE}/list-${NICK_NAME}.log"

  echo "Launch rclone"
  case $MODE in
    test)
      # test run in dry-run mode
      echo "** Test mode activated, only a dry-run will be made **"
      # NOTE : rclone old version does not support --human-readable
      #rclone "${RCLONE_METHOD}" "${RCLONE_BACKEND[@]}" "${LIST_FLAGS[@]}" --dry-run -vv --human-readable --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${TARGET}"
      rclone "${RCLONE_METHOD}" "${RCLONE_BACKEND[@]}" "${LIST_FLAGS[@]}" --dry-run -vv -P --stats-one-line --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${TARGET}"
    ;;

    *)
      # launch rclone
      # NOTE : rclone old version does not support --human-readable
      #rclone "${RCLONE_METHOD}" "${RCLONE_BACKEND[@]}" "${LIST_FLAGS[@]}" -vv --human-readable --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${TARGET}"
      rclone "${RCLONE_METHOD}" "${RCLONE_BACKEND[@]}" "${LIST_FLAGS[@]}" -vv -P --stats-one-line --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${TARGET}"
    ;;
  esac

  rm -f "${FILTERS_FILE}"
}

build_filter_file() {
  echo "Building rclone filter file from include/exclude rules with priority : ${PRIORITY}"
if [ -z "${EXCLUDE:-}" ]; then
    echo "No exclude rules defined, generating a default one to exclude nothing"
    # exclude list
    # lines beginning with # are ignored
    EXCLUDE="${__CURRENT_FILE_DIR}/exclude-${NICK_NAME}.txt"
(cat <<'EOL'
EOL
) > "${EXCLUDE}"
fi

if [ -z "${INCLUDE:-}" ]; then
  echo "No include rules defined, generating a default one to include everything"
  INCLUDE="${__CURRENT_FILE_DIR}/include-${NICK_NAME}.txt"
(cat <<'EOL'
**
EOL
) > "${INCLUDE}"
fi


  rm -f "${FILTERS_FILE}"
  if [ ! -s "${INCLUDE}" ] && [ ! -s "${EXCLUDE}" ]; then
    FILTERS_FILE=""
    return 0
  fi

  add_rules() {
    [ -s "$1" ] || return 0
    
    while IFS= read -r line || [ -n "$line" ]; do
      # ignore commentaires (#...) et lignes vides
      case "${line}" in
        ""|"#"*|";"*) continue ;;
      esac
      printf '%s %s\n' "$2" "$line" >> "${FILTERS_FILE}"
    done < "$1"
  }
  
  # - PRIORITY=include  -> includes first, then excludes (+ "wins")
  # - PRIORITY=exclude  -> excludes first, then includes (- "wins")
  if [ "${PRIORITY}" = "exclude" ]; then
    [ -s "${EXCLUDE}" ] && add_rules "${EXCLUDE}" "-"
    [ -s "${INCLUDE}" ] && add_rules "${INCLUDE}" "+"
  else
    [ -s "${INCLUDE}" ] && add_rules "${INCLUDE}" "+"
    [ -s "${EXCLUDE}" ] && add_rules "${EXCLUDE}" "-"
  fi
}
  

check() {
	if ! command -v rclone >/dev/null 2>&1; then
		local msg="ERROR: Missing rclone binary (PATH=$PATH)"
		echo "$msg" | tee -a "${WORKSPACE}/log-${NICK_NAME}.log" >&2
		exit 1
	fi
}

sync_with_tool_rclone_with_filter_sample.sh

#!/usr/bin/env bash
# https://gist.github.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801
# Synched Myrient No-Intro files
# Synched files : https://myrient.erista.me/files/No-Intro

# ./myrient_nointro_no-intro.sh "/mnt/NEON_MEDIA/VIDEO_GAMES_ROM/[ROM_ROOT]/MAIN/NOINTRO/No-Intro" test
# ./myrient_nointro_no-intro.sh "/volume1/NEON_MEDIA/VIDEO_GAMES_ROM/[ROM_ROOT]/MAIN/NOINTRO/No-Intro" test

CURRENT_FILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

usage() {
	echo "Usage : $0 <target path> [test]"
	echo "sample current usage $0 \"/volume1/NEON_MEDIA/VIDEO_GAMES_ROM/[ROM_ROOT]/MAIN/NOINTRO/No-Intro\""
}

TARGET="${1:-}"
MODE="${2:-}"

[ -z "${TARGET:-}" ] && { echo "Missing target path"; usage; exit 1; }

NICK_NAME="$(basename $0 .sh)"
RCLONE_METHOD="copy" # sync or copy
RCLONE_BACKEND=()
RCLONE_BACKEND+=( :http: )
RCLONE_BACKEND+=( --http-url="https://myrient.erista.me/files/No-Intro")
PRIORITY="include" # include or exclude
export WORKSPACE="${CURRENT_FILE_DIR}/workspace-${NICK_NAME}"
mkdir -p "$WORKSPACE"

rm -f "${WORKSPACE}/sync_with_tool_rclone_with_filter.sh"
curl -sSL "https://gist.githubusercontent.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801/raw/sync_with_tool_rclone_with_filter.sh?fake_arg_to_invalid_cache_$(date +%s)" -o "${WORKSPACE}/sync_with_tool_rclone_with_filter.sh"
. "${WORKSPACE}/sync_with_tool_rclone_with_filter.sh"

# exclude list
# lines beginning with # are ignored
EXCLUDE="${WORKSPACE}/exclude-${NICK_NAME}.txt"
(cat <<'EOL'
**
Non-Redump*/**
Unofficial*/**
Source Code*/**
IBM*/**
*(Aftermarket)*/**
EOL
) > "${EXCLUDE}"

# include list
# lines beginning with # are ignored
INCLUDE="${WORKSPACE}/include-${NICK_NAME}.txt"
(cat <<'EOL'
# Nintendo

# good folder
Nintendo - Nintendo 64 (BigEndian)/**
Nintendo - Nintendo 3DS (Decrypted)/**
Nintendo - New Nintendo 3DS (Decrypted)/**
Nintendo - Nintendo Entertainment System (Headerless)/**
Nintendo - Nintendo Game Boy/**
Nintendo - Nintendo Game Color/**
Nintendo - Nintendo Game Advance/**
Nintendo - Super Nintendo Entertainment System/**
Nintendo - Nintendo DS/**
Nintendo - Virtual Boy/**
Nintendo - Game & Watch/**

# Atari
Atari - Atari 2600/**
Atari - Atari 5200/**
Atari - Atari 7800 (BIN)/**
Atari - Atari Jaguar (J64)/**
Atari - Atari Lynx (LNX)/**

# Sega
Sega - SG-1000 - SC-3000/**
Sega - Master System - Mark III/**
Sega - Mega Drive - Genesis/**
Sega - Game Gear/**
Sega - 32X/**

# Bandai
Bandai - WonderSwan/**
Bandai - WonderSwan Color/**

# NEC
NEC - PC Engine - TurboGrafx-16/**
NEC - PC Engine SuperGrafx/**

# SNK
SNK - NeoGeo Pocket/**
SNK - NeoGeo Pocket Color/**

# VTech
VTech - CreatiVision/**
VTech - V.Smile/**

# Microsoft
Microsoft - MSX/**
Microsoft - MSX2/**

# Coleco
Coleco - ColecoVision/**

# Commodore
Commodore - Commodore 64/**
Commodore - Plus-4/**


Watara - Supervision/**
Yamaha - Copera/**
Zeebo - Zeebo/**

Sony - PlayStation Portable (PSN) (Decrypted)/**
Sony - PlayStation Portable (PSN) (Minis) (Decrypted)/**

EOL
) > "${INCLUDE}"



process_rclone_sync_with_filter