Remote sync files with various method for various protocol

-remote_sync_files.md

Remote sync/copy files with various method for various protocol

copy_with_tool_rclone_protocol_http_1fichier.sh

#!/usr/bin/env bash
# https://gist.github.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801
# 1fichier shared folder : https://1fichier.com/dir/folder_id
# NOTE : this method do not sync but use copy method of rclone. So files not existing in source will not be deleted from the destination folder

# Do this at least once to add a rclone source named "foo" of type "fichier" for the given shared_folder
# (cat <<'EOL'
# [foo]
# type = fichier
# shared_folder = folder_id
# api_key = xxxxxxxxxxxxxx
# EOL
# )>> "${HOME}/.config/rclone/rclone.conf"

CURRENT_FILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORKSPACE="${CURRENT_FILE_DIR}"
rm -f "${WORKSPACE}"/*.log


process_rclone_copy() {
   # generate a list of files to process
   rclone lsl --human-readable "${URI}" >"${WORKSPACE}/list-${NICK_NAME}.log"
    
  case $MODE in
    test)
      # test run in dry-run mode
      echo "** Test mode activated, only a dry-run will be made **"
      rclone copy --dry-run -vv --human-readable --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"
    ;;
    
    *)
      # launch copy
      # NOTE : this is a copy, not a sync. Files are not deleted from destination if not exist in source
      rclone copy -vv --human-readable --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"
    ;;
	esac
}
usage() {
	echo "Usage : $0 [test]"
}
if [ "$1" = "test" ]; then
	echo "** Test mode activated, only a dry-run will be made **"
	MODE="test"
fi
usage

check() {
	[ ! -f "${HOME}/.config/rclone/rclone.conf" ] && echo "Missing rclone conf ${HOME}/.config/rclone/rclone.conf" > "${WORKSPACE}/log-${NICK_NAME}.log" && exit 1
	if ! type -P rclone > /dev/null; then
		echo "Missing rclone binary" > "${WORKSPACE}/log-${NICK_NAME}.log" 
		exit 1
	fi
}

NICK_NAME="foo"
URI="foo:"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content"
check
process_rclone_copy

sync_with_tool_rclone_protocol_http_internetarchive.sh

#!/usr/bin/env bash
# https://gist.github.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801
# Sync file from 2 internet archive url into one folder
# Sync files : https://archive.org/details/foo1
#		           https://archive.org/details/foo2

# Do this at least once to add a rclone source named "internetarchive" of type "internetarchive"
# (cat <<'EOL'
# [internetarchive]
# type = internetarchive
# EOL
# )>> "${HOME}/.config/rclone/rclone.conf"

CURRENT_FILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORKSPACE="${CURRENT_FILE_DIR}"
rm -f "${WORKSPACE}"/*.log



process_rclone_sync() {
  # generate a list of files to process
  rclone lsl "${URI}" --human-readable --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata" >"${WORKSPACE}/list-${NICK_NAME}.log"

  case $MODE in
    test)
      # test run in dry-run mode
      echo "** Test mode activated, only a dry-run will be made **"
      rclone sync --dry-run -vv --human-readable --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata" --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${WORKSPACE}/content/${NICK_NAME}"
    ;;

    *)
      # launch sync
      rclone sync -vv --human-readable --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata" --log-file="${WORKSPACE}/sync-${NICK_NAME}.log" "${URI}" "${WORKSPACE}/content/${NICK_NAME}"
    ;;
  esac
}
usage() {
	echo "Usage : $0 [test]"
}
if [ "$1" = "test" ]; then
	echo "** Test mode activated, only a dry-run will be made **"
	MODE="test"
fi
usage

check() {
	[ ! -f "${HOME}/.config/rclone/rclone.conf" ] && echo "Missing rclone conf ${HOME}/.config/rclone/rclone.conf" > "${WORKSPACE}/log-${NICK_NAME}.log" && exit 1
	if ! type -P rclone > /dev/null; then
		echo "Missing rclone binary" > "${WORKSPACE}/log-${NICK_NAME}.log" 
		exit 1
	fi
}

# PART1
NICK_NAME="foo-part1"
# NOTE foo1 is the subpath of the internet archive url
URI="internetarchive:foo1"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content/${NICK_NAME}"

check
process_rclone_sync


# PART2
NICK_NAME="foo-part2"
# NOTE foo2 is the subpath of the internetarchive url
URI="internetarchive:foo2"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content/${NICK_NAME}"

check
process_rclone_sync

sync_with_tool_rclone_protocol_http_with_include_or_exclude.sh

#!/usr/bin/env bash
# https://gist.github.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801
# Sync files : https://myrient.erista.me/files/No-Intro

CURRENT_FILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORKSPACE="${CURRENT_FILE_DIR}"
rm -f "${WORKSPACE}"/*.log



process_rclone_sync_with_include() {
  # generate a list of files to process
  rclone lsl :http: --include-from="${INCLUDE}" --http-url="${URI}" --human-readable >"${WORKSPACE}/list-${NICK_NAME}.log"

  case $MODE in
    test)
      # test run in dry-run mode
      echo "** Test mode activated, only a dry-run will be made **"
      rclone sync :http: --dry-run -vv --human-readable --log-file="${WORKSPACE}/log-${NICK_NAME}.log" --include-from="${INCLUDE}" --http-url="${URI}" "${WORKSPACE}/content/${NICK_NAME}"
    ;;

    *)
      # launch sync
      rclone sync :http: -vv --human-readable --log-file="${WORKSPACE}/sync-${NICK_NAME}.log" --include-from="${INCLUDE}" --http-url="${URI}" "${WORKSPACE}/content/${NICK_NAME}"
    ;;
  esac
}

process_rclone_sync_with_exclude() {
  # generate a list of files to process
  rclone lsl :http: --exclude-from="${EXCLUDE}" --http-url="${URI}" --human-readable >"${WORKSPACE}/list-${NICK_NAME}.log"

  case $MODE in
    test)
      # test run in dry-run mode
      echo "** Test mode activated, only a dry-run will be made **"
      rclone sync :http: --dry-run -vv --human-readable --log-file="${WORKSPACE}/log-${NICK_NAME}.log" --exclude-from="${EXCLUDE}" --http-url="${URI}" "${WORKSPACE}/content/${NICK_NAME}"
    ;;

    *)
      # launch sync
      rclone sync :http: -vv --human-readable --log-file="${WORKSPACE}/sync-${NICK_NAME}.log" --exclude-from="${EXCLUDE}" --http-url="${URI}" "${WORKSPACE}/content/${NICK_NAME}"
    ;;
  esac
}


usage() {
	echo "Usage : $0 [test]"
}
if [ "$1" = "test" ]; then
	echo "** Test mode activated, only a dry-run will be made **"
	MODE="test"
fi
usage

check() {
	if ! type -P rclone > /dev/null; then
		echo "Missing rclone binary" > "${WORKSPACE}/log-${NICK_NAME}.log" 
		exit 1
	fi
}

NICK_NAME="myrient-nointro"
URI="https://myrient.erista.me/files/No-Intro"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content"

# exclude list
EXCLUDE="${WORKSPACE}/exclude-${NICK_NAME}.txt"
(cat <<'EOL'
Arcade - PC-based/**
Audio CD/**
CD-ROM/**
DVD-ROM/**
DVD-Video/**
HD DVD/**
EOL
)> "${EXCLUDE}"

# include list
INCLUDE="${WORKSPACE}/include-${NICK_NAME}.txt"
(cat <<'EOL'
Nintendo - Nintendo Entertainment System (Headered)/**
Nintendo - Nintendo Entertainment System (Headerless)/**
EOL
)> "${INCLUDE}"

check
process_rclone_sync_with_include
#process_rclone_sync_with_exclude

sync_with_tool_rysnc_protocol_rsync_with_include_or_exclude.sh

#!/usr/bin/env bash
# https://gist.github.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801
# Sample with Myrient No-Intro files (But rsync protocol not supported anymore since May 2024)
# Sync files : rsync://rsync.myrient.erista.me/files/No-Intro (see https://myrient.erista.me/files/No-Intro/)

CURRENT_FILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORKSPACE="${CURRENT_FILE_DIR}"
rm -f "${WORKSPACE}"/*.log


process_rsync_archive_with_exclude() {
	# generate a list of files to process
	rsync --list-only -vv --archive --stats --human-readable --recursive --exclude-from="${EXCLUDE}" --log-file="${WORKSPACE}/list-${NICK_NAME}.log" "${URI}"

	case $MODE in
		test)
			# test run in dry-run mode
			echo "** Test mode activated, only a dry-run will be made **"
			rsync --dry-run -vv --delete-before --delete-excluded --stats --human-readable --progress --recursive --sparse --exclude-from="${EXCLUDE}" --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"	
		;;

		*)
			# launch sync
			rsync -vv --archive --delete-before --delete-excluded --stats --human-readable --progress --recursive --sparse --exclude-from="${EXCLUDE}" --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"
		;;
	esac
}
process_rsync_archive_with_include() {
	# generate a list of files to process
	# NOTE rsync option '--list-only' seems to not work with '--files-from' so use a dry-run instead
	rsync --dry-run -vv --archive --stats --human-readable --recursive --files-from="${INCLUDE}" "${URI}" "${TARGET}" >"${WORKSPACE}/list-${NICK_NAME}.log"

	case $MODE in
	  test)
	    # test run in dry-run mode
	    echo "** Test mode activated, only a dry-run will be made **"
	    rsync --dry-run -vv --archive --delete-before --delete-excluded --stats --human-readable --progress --recursive --sparse --files-from="${INCLUDE}" --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"
	  ;;

	  *)
	    # launch sync
	    rsync -vv --archive --delete-before --delete-excluded --stats --human-readable --progress --recursive --sparse --files-from="${INCLUDE}" --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"
	  ;;
	esac
}
usage() {
	echo "Usage : $0 [test]"
}
if [ "$1" = "test" ]; then
	echo "** Test mode activated, only a dry-run will be made **"
	MODE="test"
fi
usage

check() {
	if ! type -P rsync > /dev/null; then
		echo "Missing rsync binary" > "${WORKSPACE}/log-${NICK_NAME}.log" 
		exit 1
	fi
}

NICK_NAME="myrient-nointro"
URI="rsync://rsync.myrient.erista.me/files/No-Intro"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content"

# exclude list
# sample exclude list to reduce size to 7Tb
# see https://myrient.erista.me/files/No-Intro/
EXCLUDE="${WORKSPACE}/exclude-${NICK_NAME}.txt"
(cat <<'EOL'
Arcade - PC-based/
Audio CD/
CD-ROM/
DVD-ROM/
DVD-Video/
HD DVD/
EOL
)> "${EXCLUDE}"

# include list
INCLUDE="${WORKSPACE}/include-${NICK_NAME}.txt"
(cat <<'EOL'
Nintendo - Nintendo Entertainment System (Headered)/
Nintendo - Nintendo Entertainment System (Headerless)/
EOL
)> "${INCLUDE}"

check
process_rsync_archive_with_include
#process_rsync_archive_with_exclude