A CLI for two-way rclone syncing with safe conflict resolution

two-way-rclone.py

#!/usr/bin/env python3
__ETAG__ = '' # NOTE: replaced by update script
__version__ = '2.0.9'
__GIST__ = 'https://gist.githubusercontent.com/u8sand/498766e540939ca2c3550c1986deb5bd/raw/two-way-rclone.py'
__README__ = '''
This utility facilitates "smart" two-way-sync between two rclone stores.
The first location is considered the "localpath" & metadata about the previous
execution is stored in this location. This information can be used to determine
if changes were made on the local side or on the remote side since the last run.

rclone-reported hashes are used for comparing file equality: disjoint change-sets
are handled seamlessly, while conflicts are handled by choosing the newest, and
backing up the oldest into a hidden file labeled by the hash of that file. This has
the benefit of being collision free.

There remains some situations where information could be lost, mainly if either
the remote or local files are modified between the time the metadata is assembled
and the time the updates are being propagated. I'll spend time in the future to
investigate whether this risk can be further reduced or eliminated.

Typical usage would have you run this script periodically or on-demand,
 the `--trash` option is recommended if you're worried about your files.

Usage:
two-way-rclone sync --trash .Trash localpath remote:path

# if you have conflicts, the review command can help you conveniently resolve them
# NOTE: this feature is in beta, it still respects trash but doesn't always work
#       in an intuitive fashion for certain types of conflicts.
two-way-rclone review --trash .Trash localpath remote:path
'''

import io
import os
import re
import sys
import json
import uuid
import time
import click
import random
import shutil
import logging
import datetime
import subprocess
import contextlib
import typing as t
from pathlib import Path, PurePosixPath

logger = logging.getLogger()

## UTILS

class PurePosixPathEx(PurePosixPath):
  ''' Additional PurePosixPath-like helper methods
  '''
  def with_parent(self, parent: PurePosixPath | str):
    return self.__class__(parent) / self.as_posix()

  def with_inserted_suffix(self, index: int, suffix: str):
    suffixes = self.suffixes
    suffixes.insert(index, suffix)
    new_suffix = ''.join(suffixes)
    return self.with_name(
      self.stem.partition('.')[0]
      + new_suffix
    )

  def without_suffix(self, suffix: str):
    new_suffix = ''.join(
      _suffix
      for _suffix in self.suffixes
      if _suffix != suffix
    )
    return self.with_name(
      self.stem.partition('.')[0]
      + new_suffix
    )

rfc3339_nano_expr = re.compile(r'^(?P<date>\d{4}-\d{2}-\d{2})T(?P<time>\d{2}:\d{2}:\d{2})\.(?P<subsecond>\d+)(?P<timezone>Z|[\+\-]\d{2}:\d{2})$')
def datetime_from_rfc3339_nano(date_string: str):
  m = rfc3339_nano_expr.match(date_string)
  if not m: raise ValueError(f"{date_string} does not match expr")
  return datetime.datetime.fromisoformat(
    f"{m.group('date')}T{m.group('time')}.{m.group('subsecond')[:6]}{'+00:00' if m.group('timezone') == 'Z' else m.group('timezone')}"
  )

def ensureBytesIO(_in):
  if type(_in) == str: return io.BytesIO(_in.encode())
  elif type(_in) == bytes: return io.BytesIO(_in)
  else: return _in

def compatible(a: dict, b: dict):
  K = a.keys() & b.keys()
  if not K: raise NotImplementedError
  return all(a[k] == b[k] for k in K)

def first(it: t.Iterable):
  return next(iter(it))

def ensureTrailingSlash(path):
  return path.rstrip('/') + '/'

def diffPaths(present, past):
  if past is None: return present.keys(), set(), set()
  added = present.keys() - past.keys()   # things here now that weren't there in the past are added
  removed = past.keys() - present.keys() # things in the past that are not in the present were removed
  modified = {
    fileId
    for fileId in (past.keys() & present.keys())                           # things in present and the past
    if not compatible(past[fileId]['Hashes'], present[fileId]['Hashes']) # no modification if hashes overlap
  }
  return added, modified, removed

def verboseOption(**kwargs):
  ''' Click option to configure logging verbosity
  '''
  class CallableFormatter:
    def __init__(self, fmt: t.Callable[[logging.LogRecord], str]):
      self._fmt = fmt

    def format(self, record):
      return self._fmt(record)

  def callback(ctx: click.Context, param: click.Parameter, value: int) -> None:
    if not value or ctx.resilient_parsing: return
    sh = logging.StreamHandler()
    sh.setFormatter(CallableFormatter(lambda r: f"{'' if r.name == 'root' else r.name}[{r.levelname.lower()}]: {r.msg}"))
    logger.addHandler(sh)
    logger.setLevel(max(logging.WARNING - value*10, logging.DEBUG))

  param_decls = ('-v', '--verbose',)
  kwargs.setdefault('count', True)
  kwargs.setdefault('type', int)
  kwargs.setdefault('default', 0)
  kwargs.setdefault('is_eager', True)
  kwargs.setdefault('help', 'Increase verbosity, more `v`s, more verbose')
  kwargs['callback'] = callback
  return click.option(*param_decls, **kwargs)

def updateOption(url: str, **kwargs):
  ''' Click option to update from url, saves ETag during update
  '''
  def callback(ctx: click.Context, param: click.Parameter, value: bool) -> None:
    if not value or ctx.resilient_parsing: return
    nonlocal url
    import urllib.request
    with urllib.request.urlopen(urllib.request.Request(url, method='HEAD')) as res:
      assert res.getcode() == 200, f"Expected 200, got {res.getcode()}"
      etag = res.getheader('ETag')
      if etag == __ETAG__:
        click.echo('Already at latest version')
        ctx.exit()
    logging.info('Saving current version...')
    current_file = Path(__file__)
    shutil.copyfile(current_file, current_file.with_stem(current_file.stem + '-' + __version__))
    logging.info('Fetching new version...')
    with urllib.request.urlopen(url) as res:
      assert res.getcode() == 200, f"Expected 200, got {res.getcode()}"
      with current_file.open('wb') as fw:
        # hashbang
        fw.write(res.readline())
        # etag
        res.readline()
        updated_etag = f"__ETAG__ = {repr(etag)}\n"
        logger.debug(updated_etag.strip())
        fw.write(updated_etag.encode()) 
        # version
        updated_version = res.readline().decode() 
        logger.debug(updated_version.strip())
        fw.write(updated_version.encode())
        # rest of the file
        shutil.copyfileobj(res, fw)
    ctx.exit()

  param_decls = ('-U', '--update',)
  kwargs.setdefault('is_flag', True)
  kwargs.setdefault('expose_value', False)
  kwargs.setdefault('is_eager', True)
  kwargs.setdefault('help', f"Update this script from {url}.")
  kwargs['callback'] = callback
  return click.option(*param_decls, **kwargs)

## RCLONE

# in the case of multiple hashes, we'll choose the one to use in this order
hashOrder = (
  'sha1',
  'md5',
  'dropbox',
  'mailru',
  'quickxor',
  'whirlpool',
  'crc32',
)

class RClone:
  ''' RClone helper -- python-helper to rclone command + some higher level methods
  '''

  def __init__(self, rclone: str):
    self.rclone = rclone.split(' ')
    self.logger = logger.getChild('rclone')
  
  def configShow(self, remote: str, _err=sys.stderr):
    self.logger.debug(f"config show {remote}")
    return subprocess.check_output(
      [*self.rclone, 'config', 'show', remote],
      stderr=_err,
      env=os.environ,
    )

  def lsjson(self, *args, _err=sys.stderr):
    self.logger.debug(f"lsjson {' '.join(args)}")
    return json.loads(
      subprocess.check_output(
        [*self.rclone, 'lsjson', *args],
        stderr=_err,
        env=os.environ,
      )
    )

  def cat(self, *args, _err=sys.stderr):
    self.logger.debug(f"cat {' '.join(args)}")
    return subprocess.check_output(
      [*self.rclone, 'cat', *args],
      stderr=_err,
      env=os.environ,
    ).decode()

  def copy(self, *args, _in=None, _err=sys.stderr):
    self.logger.debug(f"copy {' '.join(args)} < {id(_in)=}")
    proc = subprocess.Popen([*self.rclone, 'copy', *args],
      stdin=subprocess.PIPE if _in is not None else None,
      stdout=sys.stdout,
      stderr=_err,
      env=os.environ,
    )
    if _in is not None:
      shutil.copyfileobj(ensureBytesIO(_in), proc.stdin)
      proc.stdin.close()
    returncode = proc.wait()
    if returncode != 0: raise subprocess.CalledProcessError(returncode, f"{self.rclone} {' '.join(args)}")
    return returncode

  def delete(self, *args, _in=None, _err=sys.stderr):
    self.logger.debug(f"delete {' '.join(args)} < {id(_in)=}")
    proc = subprocess.Popen(
      [*self.rclone, 'delete', *args],
      stdin=subprocess.PIPE if _in is not None else None,
      stdout=sys.stdout,
      stderr=_err,
      env=os.environ,
    )
    if _in is not None:
      shutil.copyfileobj(ensureBytesIO(_in), proc.stdin)
      proc.stdin.close()
    returncode = proc.wait()
    if returncode != 0: raise subprocess.CalledProcessError(returncode, f"{self.rclone} {' '.join(args)}")
    return returncode

  def deletefile(self, *args, _err=sys.stderr):
    self.logger.debug(f"deletefile {' '.join(args)}")
    return subprocess.check_call(
      [*self.rclone, 'deletefile', *args],
      stdout=sys.stdout,
      stderr=_err,
      env=os.environ,
    )

  def rmdirs(self, *args, _err=sys.stderr):
    self.logger.debug(f"rmdirs {' '.join(args)}")
    return subprocess.check_call(
      [*self.rclone, 'rmdirs', *args],
      stdout=sys.stdout,
      stderr=_err,
      env=os.environ,
    )

  def mkdir(self, *args, _err=sys.stderr):
    self.logger.debug(f"mkdir {' '.join(args)}")
    return subprocess.check_call(
      [*self.rclone, 'mkdir', *args],
      stdout=sys.stdout,
      stderr=_err,
      env=os.environ,
    )

  def moveto(self, *args, _err=sys.stderr):
    self.logger.debug(f"moveto {' '.join(args)}")
    return subprocess.check_call(
      [*self.rclone, 'moveto', *args],
      stdout=sys.stdout,
      stderr=_err,
      env=os.environ,
    )

  def rcat(self, *args, _in=None, _err=sys.stderr):
    self.logger.debug(f"rcat {' '.join(args)} < {id(_in)=}")
    proc = subprocess.Popen([*self.rclone, 'rcat', *args],
      stdin=subprocess.PIPE if _in is not None else None,
      stdout=sys.stdout,
      stderr=_err,
      env=os.environ,
    )
    if _in is not None:
      shutil.copyfileobj(ensureBytesIO(_in), proc.stdin)
      proc.stdin.close()
    returncode = proc.wait()
    if returncode != 0: raise subprocess.CalledProcessError(returncode, f"{self.rclone} {' '.join(args)}")
    return returncode

  def remoteOf(self, path: str):
    ''' A wapper fs like crypt stores the inner remote as `remote = something:
    this helper will resolve the inner remote and path based on the outer path by checking
    the outer remote's config.
    '''
    remote, sep, remote_path = path.partition(':')
    if sep is None: return path
    m = re.search(r'^remote = (.+)$', self.configShow(remote), re.MULTILINE)
    if not m: raise NotImplementedError
    inner_remote, sep, inner_remote_path = m.group(1).partition(':')
    if sep:
      return f"{inner_remote}:{PurePosixPath.joinpath(inner_remote_path, remote_path)}"
    else:
      return f"{PurePosixPath.joinpath(inner_remote, inner_remote_path, remote_path)}"

  def present(self, path, encrypted=False, localPast=None):
    if encrypted:
      # rclone lsjson on crypt remotes does not contain hashes even with --hash
      #  so instead we get hashes from the raw store and insert them into the
      #  encrypted lsjson metadata
      present = {}
      encrypted_path_to_path = {}
      for file in self.lsjson(
        '--recursive', '--files-only',
        '--encrypted', '--no-mimetype', path
      ):
        present[file['Path']] = file if localPast is None else dict(localPast.get(file['Path'], {}), **file)
        encrypted_path_to_path[file['EncryptedPath']] = file['Path']
      backend_path = self.remoteOf(path)
      for file in self.lsjson(
        '--recursive', '--files-only',
        '--hash', '--no-mimetype', backend_path
      ):
        present[encrypted_path_to_path[file['Path']]].update(Hashes=file['Hashes'])
      return present
    else:
      return {
        file['Path']: file if localPast is None else dict(localPast.get(file['Path'], {}), **file)
        for file in self.lsjson(
          '--recursive', '--files-only',
          '--hash', '--no-mimetype', path
        )
      }

  @contextlib.contextmanager
  def lock(self, lockfile, dry_run=False):
    self.logger.debug(f"lock {lockfile}")
    if not dry_run:
      lockid = uuid.uuid4().hex
      self.rcat(lockfile, _in=lockid)
      time.sleep(0.1 + random.random()/2)
      assert self.cat(lockfile) == lockid, 'Failed to verify lock was acquired'
    try:
      yield
    finally:
      if not dry_run:
        self.deletefile(lockfile)


## COMMANDS

@click.group(help=__README__)
@click.version_option(__version__, '-V', '--version')
@updateOption(__GIST__)
@verboseOption()
def cli(**kwargs): pass

@cli.command(help='Two-way sync with with conflict preservation strategy')
@click.option('-b', '--bin', envvar='TWO_WAY_RCLONE_BIN', default='rclone', help='rclone binary')
@click.option('-c', '--cache', envvar='TWO_WAY_RCLONE_CACHE', default='.two-way-rclone-cache/', help='Two way Rclone cache prefix')
@click.option('-n', '--no-cache', envvar='TWO_WAY_RCLONE_NO_CACHE', is_flag=True, default=False, help='Disable cache (not recommended)')
@click.option('-d', '--dry-run', envvar='TWO_WAY_RCLONE_DRY_RUN', is_flag=True, help='Run and report what would have happend without doing it')
@click.option('-f', '--force', envvar='TWO_WAY_RCLONE_FORCE', is_flag=True, help='Force run even if lock files already exist')
@click.option('-t', '--trash', envvar='TWO_WAY_RCLONE_TRASH', default=None, help='Move files to trash instead of deleting')
@click.option('-e', '--encrypted', envvar='TWO_WAY_RCLONE_ENCRYPTED', is_flag=True, default=False, help='Use this flag when operating with crypt remotes')
@verboseOption()
@click.argument('localpath')
@click.argument('remotepath')
def sync(localpath, remotepath, bin='rclone', cache='.two-way-rclone-cache/', no_cache=False, dry_run=False, force=False, trash=None, encrypted=False, **kwargs):
  rclone = RClone(bin)
  localpath = ensureTrailingSlash(localpath)
  remotepath = ensureTrailingSlash(remotepath)
  cache = ensureTrailingSlash(cache)
  if trash: trash = ensureTrailingSlash(trash)
  logger.info(f"Syncing {repr(localpath)} <=> {repr(remotepath)} {'(dry-run)' if dry_run else ''}")
  # here we load the "localPast" -- a saved json of the local state when we last ran this command
  logger.info('Checking last state..')
  if no_cache: localPast = None
  else:
    try:
      localPast = {
        file['Path']: file
        for file in json.loads(rclone.cat(
          f"{localpath}{cache}last.local.json",
          _err=subprocess.DEVNULL,
        ))
      }
    except subprocess.CalledProcessError as e:
      if e.returncode != 3: raise
      localPast = None

  # we load the "localPresent" -- the json of the local state now, mkdir in case this is the first time
  logger.info('Checking current state..')
  rclone.mkdir(localpath)
  localPresent = rclone.present(localpath, localPast=localPast)
  # now check for a lock to prevent running this command twice on the same system which could result
  #  in logic collisions and potentially data loss
  if not force and f"{cache}current.lock" in localPresent:
    logger.error('Lock exists locally, two-way-clone likely in progress, if not run with --force')
    sys.exit(1)

  # we load the "remotePresent" -- the json of the remote state now, mkdir in case this is the first time
  logger.info('Checking remote state..')
  rclone.mkdir(remotepath)
  remotePresent = rclone.present(remotepath, encrypted=encrypted)
  # now check for a lock on the remote to prevent running this command twice on the same system which could result
  #  in logic collisions and potentially data loss. Notably, if something else modifies the remote
  #  that isn't this script, you can still have an issue, so be aware that this isn't fullproof.
  if not force and f"{cache}current.lock" in remotePresent:
    logger.error('Lock exists in remote, two-way-clone likely in progress, if not run with --force')
    sys.exit(1)

  # make the actual locks locally and remotely and hold them for the duration of the sync
  logger.info('Acquiring local lock..')
  with rclone.lock(f"{localpath}{cache}current.lock", dry_run=dry_run):
    logger.info('Acquiring remote lock..')
    with rclone.lock(f"{remotepath}{cache}current.lock", dry_run=dry_run):
      logger.info('Diffing state..')
      # ignore "present" files in the cache/trash
      for fileId in list(localPresent):
        if fileId.startswith(cache) or (trash and fileId.startswith(trash)):
          del localPresent[fileId]

      for fileId in list(remotePresent):
        if fileId.startswith(cache) or (trash and fileId.startswith(trash)):
          del remotePresent[fileId]

      # gather changes (differences from the last time we ran this i.e. "localPast")
      localAdded, localModified, localRemoved = diffPaths(localPresent, localPast)
      remoteAdded, remoteModified, remoteRemoved = diffPaths(remotePresent, localPast)
      conflicts = (localModified & remoteRemoved) | (remoteModified & localRemoved) | {    # modified in one place & removed in the other
        fileId
        for fileId in ((localAdded & remoteAdded) | (localModified & remoteModified))      # added or modified in both places
        if not compatible(localPresent[fileId]['Hashes'], remotePresent[fileId]['Hashes']) # no conflict if hashes overlap
      }
      # files removed from both locations should be eliminated entirely
      commonRemoved = localRemoved & remoteRemoved
      localRemoved -= commonRemoved
      remoteRemoved -= commonRemoved

      logger.info(f"{localAdded=}, {localModified=}, {localRemoved=}")
      logger.info(f"{remoteAdded=}, {remoteModified=}, {remoteRemoved=}")
      logger.info(f"{conflicts=}")

      # create plan to address the differences, each one happens in order
      toMoveRemote0 = []
      toMoveLocal1 = []
      toUpload2 = (localAdded | localModified) - conflicts
      toDownload3 = (remoteAdded | remoteModified) - conflicts
      toRemoveLocal4 = remoteRemoved - conflicts
      toRemoveRemote5 = localRemoved - conflicts

      # gather the current remote file information for anything we're meant to download from the remote
      for fileId in toDownload3:
        localPresent[fileId] = remotePresent[fileId]

      # register conflict metadata
      for fileId in conflicts:
        if fileId in remoteRemoved:
          hashType = first(
            hash for hash in hashOrder
            if hash in localPresent[fileId]['Hashes']
          )
          hash = localPresent[fileId]['Hashes'][hashType]
          path = PurePosixPathEx(fileId).without_suffix(f".{hash}")
          conflictFileId = str(path.with_inserted_suffix(-1, f".{hash}"))
          # rename the local file
          toMoveLocal1.append((fileId, conflictFileId))
          # upload the local conflict
          toUpload2.add(conflictFileId)
          # update local pathinfo
          localPresent[conflictFileId] = dict(
            localPresent[fileId],
            Path=conflictFileId,
            ConflictWith=fileId,
            ConflictType='remoteRemoved',
            ConflictResolution='saveModified',
          )
          del localPresent[fileId]
        elif fileId in localRemoved:
          hashType = first(
            hash for hash in hashOrder
            if hash in remotePresent[fileId]['Hashes']
          )
          hash = remotePresent[fileId]['Hashes'][hashType]
          path = PurePosixPathEx(fileId).without_suffix(f".{hash}")
          conflictFileId = str(path.with_inserted_suffix(-1, f".{hash}"))
          # rename the remote file
          toMoveRemote0.append((fileId, conflictFileId))
          # download the remote version
          toDownload3.add(conflictFileId)
          # update local pathinfo
          localPresent[conflictFileId] = dict(
            remotePresent[fileId],
            Path=conflictFileId,
            ConflictWith=fileId,
            ConflictType='localRemoved',
            ConflictResolution='saveModified',
          )
        else:
          localPresentTs = datetime_from_rfc3339_nano(localPresent[fileId]['ModTime'])
          remotePresentTs = datetime_from_rfc3339_nano(remotePresent[fileId]['ModTime'])
          hashType = first(
            hash for hash in hashOrder
            if hash in localPresent[fileId]['Hashes'] and hash in remotePresent[fileId]['Hashes']
          )
          if localPresentTs < remotePresentTs:
            # our modification is older
            hash = localPresent[fileId]['Hashes'][hashType]
            path = PurePosixPathEx(fileId).without_suffix(f".{hash}")
            conflictFileId = str(path.with_inserted_suffix(-1, f".{hash}"))
            # rename the local file
            toMoveLocal1.append((fileId, conflictFileId))
            # upload the local conflict
            toUpload2.add(conflictFileId)
            # download the remote version
            toDownload3.add(fileId)
            # update local pathinfo
            localPresent[conflictFileId] = dict(
              localPresent[fileId],
              Path=conflictFileId,
              ConflictWith=fileId,
              ConflictType='bothModified',
              ConflictResolution='saveModifiedUseRemote',
            )
            localPresent[fileId] = remotePresent[fileId]
          else:
            # our modification is newer
            hash = remotePresent[fileId]['Hashes'][hashType]
            path = PurePosixPathEx(fileId).without_suffix(f".{hash}")
            conflictFileId = str(path.with_inserted_suffix(-1, f".{hash}"))
            # rename the remote file
            toMoveRemote0.append((fileId, conflictFileId))
            # upload the local file
            toUpload2.add(fileId)
            # download the remote version
            toDownload3.add(conflictFileId)
            # update local pathinfo
            localPresent[conflictFileId] = dict(
              remotePresent[fileId],
              Path=conflictFileId,
              ConflictWith=fileId,
              ConflictType='bothModified',
              ConflictResolution='saveModifiedUseLocal',
            )
      #
      # move to trash instead of deleting
      if trash:
        for fileId in toRemoveLocal4:
          hashType = first(
            hash for hash in hashOrder
            if hash in localPresent[fileId]['Hashes']
          )
          hash = localPresent[fileId]['Hashes'][hashType]
          path = PurePosixPathEx(fileId).without_suffix(f".{hash}")
          newPath = str(path.with_inserted_suffix(-1, f".{hash}").with_parent(trash))
          toMoveLocal1.append((fileId, newPath))
        for fileId in toRemoveRemote5:
          hashType = first(
            hash for hash in hashOrder
            if hash in remotePresent[fileId]['Hashes']
          )
          hash = remotePresent[fileId]['Hashes'][hashType]
          path = PurePosixPathEx(fileId).without_suffix(f".{hash}")
          newPath = str(path.with_inserted_suffix(-1, f".{hash}").with_parent(trash))
          toMoveRemote0.append((fileId, newPath))

      # remove locally removed files from pathinfo
      for fileId in toRemoveLocal4:
        del localPresent[fileId]
      
      if trash:
        toRemoveLocal4 = set()
        toRemoveRemote5 = set()
      #
      logger.info('Syncing..')
      opts = ['--verbose']
      if dry_run: opts.append('--dry-run')
      #
      if toMoveRemote0:
        for sourceFileId, targetFileId in toMoveRemote0:
          rclone.moveto(*opts, f"{remotepath}{sourceFileId}", f"{remotepath}{targetFileId}")
        rclone.rmdirs(*opts, '--leave-root', remotepath)
      #
      if toMoveLocal1:
        for sourceFileId, targetFileId in toMoveLocal1:
          rclone.moveto(*opts, f"{localpath}{sourceFileId}", f"{localpath}{targetFileId}")
        rclone.rmdirs(*opts, '--leave-root', localpath)
      #
      if toUpload2: rclone.copy(
        *opts, localpath, remotepath,
        '--include-from=-', _in='\n'.join(toUpload2),
      )
      #
      if toDownload3: rclone.copy(
        *opts, remotepath, localpath,
        '--include-from=-', _in='\n'.join(toDownload3),
      )
      #
      if toRemoveLocal4: rclone.delete(
        *opts, '--rmdirs', localpath,
        '--include-from=-', _in='\n'.join(toRemoveLocal4),
      )
      if toRemoveRemote5: rclone.delete(
        *opts, '--rmdirs', remotepath,
        '--include-from=-', _in='\n'.join(toRemoveRemote5),
      )
      #
      logging.info(f"Saving new state..")
      if not dry_run:
        # update last.local.json
        if not no_cache: rclone.rcat(f"{localpath}{cache}last.local.json", _in=json.dumps(list(localPresent.values())))
      logging.info(f"Cleaning up..")
  logging.info(f"Done!")

@cli.command(help='Review & resolve conflicts')
@click.option('-b', '--bin', envvar='TWO_WAY_RCLONE_BIN', default='rclone', help='rclone binary')
@click.option('-c', '--cache', envvar='TWO_WAY_RCLONE_CACHE', default='.two-way-rclone-cache/', help='Two way Rclone cache prefix')
@click.option('-d', '--dry-run', envvar='TWO_WAY_RCLONE_DRY_RUN', is_flag=True, help='Run and report what would have happend without doing it')
@click.option('-e', '--editor', envvar='TWO_WAY_RCLONE_EDITOR', default='vimdiff', help='Editor to use for diffing')
@click.option('-t', '--trash', envvar='TWO_WAY_RCLONE_TRASH', default=None, help='Move files to trash instead of deleting')
@verboseOption()
@click.argument('localpath')
@click.argument('remotepath', required=False)
@click.pass_context
def review(ctx, localpath, remotepath, bin='rclone', cache='.two-way-rclone-cache/', editor='vimdiff', dry_run=False, trash=None, **kwargs):
  # TODO: lock/record changes
  rclone = RClone(bin)
  localpath = ensureTrailingSlash(localpath)
  cache = ensureTrailingSlash(cache)
  if trash: trash = ensureTrailingSlash(trash)
  try:
    files = {
      file['Path']: file
      for file in json.loads(rclone.cat(
        f"{localpath}{cache}last.local.json",
        _err=subprocess.DEVNULL,
      ))
    }
  except subprocess.CalledProcessError as e:
    if e.returncode != 3: raise
    files = {}
  #
  opts = ['--verbose']
  if dry_run: opts.append('--dry-run')
  #
  requires_update = False
  quitting = False
  for file in files.values():
    if 'ConflictWith' not in file: continue
    fileId = file['ConflictWith']
    conflictType = file['ConflictType']
    conflictResolution = file['ConflictResolution']
    chosenFile =  files.get(file['ConflictWith'])
    hashType = first(
      hash for hash in hashOrder
      if hash in file['Hashes'] and (chosenFile is None or hash in chosenFile['Hashes'])
    )
    logger.warning(f"{fileId}: had conflict {conflictType} and was resolved with {conflictResolution}")
    logger.warning(f" {file['ModTime']} {file['Path']}")
    logger.warning(f" {chosenFile['ModTime'] if chosenFile else 'removed'} {fileId}")
    while True:
      response = click.prompt(
        f" :: (V)iew, (S)kip, (A)ccept conflict resolution, (R)everse conflict resolution, (Q)uit: [v/s/a/r/q] ",
        type=click.Choice(['v', 's', 'a', 'r', 'q'], case_sensitive=False),
        show_choices=False,
      )
      if response == 'v':
        p = Path(localpath)
        if chosenFile:
          subprocess.call([editor, str(p/file['Path']), str(p/chosenFile['Path'])], stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr)
        else:
          subprocess.call([editor, str(p/file['Path'])], stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr)
        requires_update = True
        continue
      elif response == 's':
        pass
      elif response == 'a':
        if trash:
          rclone.moveto(*opts, f"{localpath}{file['Path']}", f"{localpath}{str(PurePosixPath(trash)/file['Path'])}")
        else:
          rclone.delete(
            *opts, '--rmdirs', localpath, 
            '--include-from=-', _in=file['Path'],
          )
        requires_update = True
      elif response == 'r':
        if chosenFile:
          if trash:
            hashType = first(
              hash for hash in hashOrder
              if hash in chosenFile['Hashes']
            )
            hash = chosenFile['Hashes'][hashType]
            path = PurePosixPathEx(chosenFile['Path']).without_suffix(f".{hash}")
            newPath = str(path.with_inserted_suffix(-1, f".{hash}").with_parent(path.parent / trash))
            rclone.moveto(*opts, f"{localpath}{chosenFile['Path']}", f"{localpath}{newPath}")
          else:
            rclone.delete(
              *opts, '--rmdirs', localpath, 
              '--include-from=-', _in=chosenFile['Path'],
            )
        #
        rclone.moveto(*opts, f"{localpath}{file['Path']}", f"{localpath}{fileId}")
        requires_update = True
      elif response == 'q':
        quitting = True
      break
    if quitting: break
  #
  if requires_update and not dry_run:
    click.echo(f"If changes were made, you should re-sync.")
    if remotepath:
      if click.confirm("Resync now?", default=True):
        ctx.forward(sync)

if __name__ == "__main__":
  cli()