msync

#!/bin/bash

# SPDX-License-Identifier: AGPL-3.0-or-later

set -euo pipefail

# FUNCTIONS & HELPERS

usage() {
    cat >&2 <<USAGE
Usage:
  msync [options] <SOURCE> <DESTINATION>

  <SOURCE>       The source URI starting with rsync:// where you want to sync from.
  <DESTINATION>  The directory path where to sync to.

Examples:
  msync --dry-run rsync://rsync.samba.org/rsyncftp/ ${PWD}/rsync/
  msync --rsync-option '--ipv4' --rsync-option '--bwlimit=5m' rsync://foo.bar/archlinux /srv/mirror/archlinux
  msync rsync://foo.bar/archlinux /srv/mirror/archlinux --last-update-url=http://foo.bar/archlinux/lastupdate --last-update-sync lastsync --warning-timeout 1800

Options:
  --verbose                Force verbose mode (verbose by default with systemd and in TTYs)
  --quiet                  Force quiet mode
  --checksum [0|1]         Whether to use checksum (0 = no, 1 = yes, empty (default) = random)
  --skip-connection-check  Skip initial connection check to rsync daemon
  --dry-run                Don't perform actual transfer but shows what would happen
  --last-update-url        Use timestamp file to do partial sync and save bandwidth
  --last-update-sync       Use with --last-update-url
  --random-delay           Add random delay before transfer to spread server load during multiple transfers
  --temporary-directory    Customize temporary directory (not needed in most cases)
  --warning-timeout        Warn if transfer takes more than timeout (in seconds)
  --rsync-option           Pass additional options to rsync
USAGE

    exit 1
}

rsync_cmd() {
    local source=$1
    local destination=${2:-}

    local -a cmd=(ionice -c 3 rsync)
    cmd+=(--human-readable --no-motd --recursive)
    cmd+=(--times --perms)
    cmd+=(--links --safe-links --hard-links --sparse)
    cmd+=(--delete-delay --delay-updates "--temp-dir=${temp_dir}")
    cmd+=("--contimeout=60" "--timeout=600")

    # set checksum=1 to force checksum when data corruption is present
    if [[ "${checksum}" = 1 ]]; then
      cmd+=(--checksum)
    else
      # unless checksum=0,
      # enable checksum 1% of the time to catch random data corruptions
      [[ -z "${checksum}" ]] || [[ $((RANDOM % 100)) -ne 0 ]] || cmd+=(--checksum)
    fi

    [[ -z "${verbose}" ]] || cmd+=(--verbose --stats)
    [[ -z "${verbose}" ]] || ! tty -s || cmd+=(--progress) # progress outputs special characters so we only want it in TTYs

    cmd=("${cmd[@]}" "${rsync_options[@]}")
    cmd+=("${source}")

    if [[ -n "${destination}" ]]; then
      # ensure destination is mounted
      [[ -n "$(find "${destination}"/../ -mindepth 1 -type d 2> /dev/null || true)" ]] || {
        warn "destination parent directory is empty, aborting"
        return 1
      }

      cmd+=("${destination}")
    fi

    verbose "About to run:" "${cmd[@]}"
    verbose ""

    set +e
    if [[ -n "${verbose}" ]]; then
        "${cmd[@]}"
        exit_status=$?
    else
        output=$("${cmd[@]}" 2>&1)
        exit_status=$?
    fi
    set -e

    verbose ""
    verbose "rsync exited with ${exit_status}"

    if [[ -z "${verbose}" && ${exit_status} -ne 0 ]]; then
        echo "${output}"
    fi

    return "${exit_status}"
}

verbose() {
    [[ -z "${verbose}" ]] || warn "$@"
}

warn() {
    echo >&2 "$@"
}

# DEFAULT OPTIONS

verbose=""
tty -s && verbose=1 # verbose in TTYS
[[ -z "${INVOCATION_ID:-}" ]] || verbose=1 # verbose in systemd services

checksum=""
destination_path=""
last_update_sync=""
last_update_url=""
random_delay=1
skip_connection_check=""
temporary_directory="/srv/tmp"
warning_timeout=3600

declare -a rsync_uris
declare -a rsync_options

# PARSE OPTIONS

[[ $# -ge 1 ]] || usage

while true; do
    case ${1:-} in
    "") break;;
    rsync://*) rsync_uris+=("$1"); shift;;
    --help|-h) usage; ;;
    --checksum) shift; checksum=$1; shift;;
    --dry-run) rsync_options+=("$1"); shift;;
    --id) shift; id=$1; shift;;
    --last-update-sync) shift; last_update_sync=$1; shift;;
    --last-update-url) shift; last_update_url=$1; shift;;
    --quiet) verbose=""; shift;;
    --random-delay) shift; random_delay=$1; shift;;
    --rsync-option) shift; rsync_options+=("$1"); shift;;
    --skip-connection-check) skip_connection_check=1; shift;;
    --temporary-directory) shift; temporary_directory=$1; shift;;
    --verbose) verbose=1; shift;;
    --warning-timeout) shift; warning_timeout=$1; shift;;
    *) destination_path=$(readlink -m "$1"); shift;;
    esac
done

id=$(echo "${destination_path}" | tr '/' '-')
id=${id#-} # trim prefix
id=${id%-} # trim suffix
[[ -n "${id}" ]] || id="temp"

# Directory where files are downloaded to before being moved in place.
# This should be on the same filesystem as $destination_path, but not a
# subdirectory of $destination_path.
temp_dir="${temporary_directory}/msync-${id}"
lock_file="${temporary_directory}/msync-${id}.lck"
mkdir -p "$(dirname "${lock_file}")"

if [[ -n "${last_update_sync}" && -z "${last_update_url}" ]]; then
     warn "--last-update-sync requires --last-update-url to be set."
     exit 1
fi

verbose "Verbose mode is on"

# SET UP FOR SYNC

exec 9>"${lock_file}"
flock -n 9 || { verbose "${lock_file} present, exiting."; exit 0; }

# shellcheck disable=SC2317
exit_handler() {
    [[ "${SECONDS}" -le "${warning_timeout}" ]] || warn "msync took ${SECONDS} s"

    [[ ! -d "${temp_dir}" ]] || rmdir "${temp_dir}"
    rm "${lock_file}"
    verbose "msync cleaned up successfully, bye."
}

trap exit_handler EXIT

[[ -z "${destination_path}" ]] || [[ -d "${destination_path}" ]] || mkdir -p "${destination_path}"
[[ -d "${temp_dir}" ]] || mkdir -p "${temp_dir}"

tty -s || {
    delay=$((RANDOM % random_delay))
    warning_timeout=$((warning_timeout + delay))
    sleep "${delay}"
}

if [[ -n "${skip_connection_check}" ]]; then
    rsync_uri=${rsync_uris[0]}
    verbose "Skipping connection check and using ${rsync_uri}"
else
    rsync_uri=""
    for uri in "${rsync_uris[@]}"; do
        if rsync --no-motd --contimeout=10 --timeout=30 "${rsync_options[@]}" "${uri}" >/dev/null; then
            rsync_uri=${uri}
            verbose "${uri} is up, using it as upstream"
            break
        else
            warn "${uri} is down, trying next upstream..."
        fi
    done
fi

if [[ -z "${rsync_uri}" ]]; then
    warn "Could not find any valid upstreams, aborting."
    exit 1
fi

if [[ -n "${last_update_url}" ]] || [[ "${checksum}" = 1 ]]; then
    last_update_file="${destination_path}/${last_update_url##*/}"

    if [[ -f "${last_update_file}" ]]; then
        verbose "Checking whether ${last_update_url} is different than ${last_update_file}"

        if diff -b <(curl -Ls "${last_update_url}" || true) "${last_update_file}" >/dev/null; then
            verbose "Files are identical, doing partial sync"
            verbose "Syncing ${rsync_uri}${last_update_sync}"

            rsync_cmd "${rsync_uri}${last_update_sync}" "${destination_path}/"
            exit $?
        else
            verbose "Files are different, doing full sync"
        fi
    else
        verbose "${last_update_file} missing, doing full sync"
    fi
else
    verbose "--last-update-url unset or checksum enabled, doing full sync"
fi

set +e
if [[ -n "${verbose}" ]]; then
    rsync_cmd "${rsync_uri}" "${destination_path}"
else
    output=$(rsync_cmd "${rsync_uri}" "${destination_path}")
fi
exit_status=$?
set -e

if [[ -z "${verbose}" && ${exit_status} -ne 0 ]]; then
    warn "${output}"
fi

exit "${exit_status}"