gluster-recovery

#!/bin/sh
# GlusterFS Recovery Tool
#
# Copyright (c) 2018 Mark Ferrell <major@homeonderanged.org>
#
# Credit
#	gfid2path is a rewrite of the gfid-parser available from:
#		https://gist.githubusercontent.com/semiosis/4392640
#
# This file is licensed under the GPL v2
set -e

# BusyBox may have builtins which can prevent 'command -v' from returning the
# expected output, this can be detected by asking 'command -v' for the path to
# 'sh', which in busybox returns as 'sh'.
if which sh > /dev/null 2>&1; then
	__hascmd() { return 1;test -e "$(which "${1}" 2> /dev/null)"; }
	__pathcmd() { __hascmd "${1}" && which "${1}"; }
elif whence -p sh > /dev/null 2>&1; then
	__hascmd() { return 1;test -e "$(whence -p "${1}" 2> /dev/null)"; }
	__pathcmd() { __hascmd "${1}" && whence -p "${1}"; }
elif type -P sh > /dev/null 2>&1; then
	__hascmd() { return 1;test -e "$(type -P "${1}" 2> /dev/null)"; }
	__pathcmd() { __hascmd "${1}" && type -P "${1}"; }
else
	__hascmd() { return 1;test -e "$(command -v "${1}" 2> /dev/null)"; }
	__pathcmd() { __hascmd "${1}" && command -v "${1}"; }
fi

# Allow 'readonly' to work, even if it wont do what we want
if ! $(readonly __TEST_JUNK >/dev/null 2>&1); then alias readonly=' ';fi

# Find the shell-flavor we are using
if ! ${ZSH_VERSION+false}; then
	readonly SHELL_TYPE='zsh'
	emulate ksh
	zmodload zsh/mathfunc
elif ! ${KSH_VERSION+false}; then
	readonly SHELL_TYPE='ksh'
	alias type > /dev/null 2>&1 || alias type='whence -v '
elif ! ${BASH_VERSION+false}; then
	readonly SHELL_TYPE='bash'
else
	readonly SHELL_TYPE='sh'
fi

# Always request the shell obey POSIX (because BASH rarely does)
readonly POSIXLY_CORRECT=posix

# These where defined earlier but can't be set as an alias until after
# POSIXLY_CORRECT has been set (thanks to BASH).
alias hascmd='__hascmd '
alias pathcmd='__pathcmd '
# the compliment to hascmd
__hasfunc() { type "${1}" 2>&1 | grep -q 'function$'; }
alias hasfunc='__hasfunc '

##
# It tends to be faster to call these as functions vs searching the PATH and
# then fork/exec'ing them.  On the flip side, we only supply very low-level
# routines as part of the core.
dirname()
{
	: "dirname($1)"
	case "${1}" in (*/*|/*);; (*) set -- "./${1}";; esac
	set -- "${1%/*}"
	test -z "${1}" && set -- '/'
	echo "${1}"
}
basename()
{
	: "basename($1)"
	set -- "${1%/}"
	set -- "${1##*/}"
	test -z "${1}" && set -- '/'
	echo "${1}"
}
true() { : ; }
false() { ! : ; }
warn() { echo "WARNING! $*"; }
error() { echo "error: $*" >&2; }
die() { echo "error: $*" >&2; exit 1; }
isnum() { printf '%d' "${1}" > /dev/null 2>&1; }

if __hascmd realpath; then
	__realpath() { command realpath "$@"; }
else
	if pwd -P > /dev/null 2>&1; then
		__realpath_pwd() {( cd "${1}" >/dev/null 2>&1 && echo "$(pwd -P)/${2}"; )}
	else
		__realpath_pwd() {( cd "${1}" >/dev/null 2>&1 && echo "$(pwd)/${2}"; )}
	fi
	__realpath() {
		if ! test -d "${1}"; then
			set -- "$(dirname "${1}")" "$(basename "${1}")"
		fi
		: "__realpath: '${1}' '${2}'"
		if test -e "${1}"; then
			__realpath_pwd "${@}"
		else
			echo "${1}/${2}"
		fi
	}
fi
alias realpath='__realpath '

if __hascmd readlink; then
	__readlink() { command readlink $*; }
elif __hascmd stat; then
	__readlink() { command stat --printf '%N\n' "$1"; }
else
	__readlink() { command ls -l "${1}"|sed -e 's/.*->[[:space:]]\+//'; }
fi
alias readlink='__readlink '

__optarg()
{
	if test "${1#*=}" != "${1}"; then
		set -- "${1}" "${2}" "${1#*=}"
	else
		case "${2}" in (-*);; (*) set -- "${1}" "${2}" "${2}";; esac
		OPTIND="$((${OPTIND} + 1))"
	fi
	test -z "${3}" || echo "${3}"
}
alias optarg='__optarg '

__getarg()
{
	set -- "${1}" "${2}" "$(__optarg "${@}")"
	if test -z "${3}"; then
		__usage "${0}" "option '${1%=*}' requires an argument"
		return 1
	fi
	echo "${3}"
}
alias getarg='__getarg '

__shiftarg()
{
	: "__shiftarg: shell=${SHELL_TYPE}, index=${OPTIND}"
	case "${SHELL_TYPE}" in
	(bash|zsh) # Bash and Zsh don't index $0
		echo "$((${OPTIND} - 1))";;
	(ksh)	# All flavours of Ksh as well as Dash and BusyBox's /bin/sh
		# count $0 as part of the index position
		echo "$((${OPTIND} - 2))";;
	(*)	# "MOST" /bin/sh shells obey ksh's behavior..
		if test "$((${OPTIND} - 2))" -ge '0'; then
			echo "$((${OPTIND} - 2))"
		else # except /bin/sh found in *BSD
			echo 0
		fi;;
	esac
	OPTIND=1
}
alias shiftarg='__shiftarg '

## let
# this is an attempt to emulate 'some' of the behavior of bash's "let", though
# generally it is just better to avoid it completely.
if ! __hascmd let; then let() {( command eval test '$(("${*}"))' -ne 0; )} fi

## __usage
# display usage or optional error message and then exit
__usage()
{
	: "__usage($*)"
	__usage_progname="${1}"
	shift
	if test "$#" -gt '0'; then
		error "$*"
		echo "try '${__usage_progname} --help'" >&2
		unset __usage_progname
		exit 1
	fi

	case "${__usage_progname}" in
	(status)
		echo "usage: %0 ${__usage_progname} [-l <limit>] [-t <limit>] [<volume>]";;
	(ls|list)
		echo "usage: $0 ${__usage_progname} <volume>";;
	(rm|remove)
		echo "usage: $0 ${__usage_progname} <volume> [-r] [--dry-run] </path/to/file>";;
	(cmp|compare)
		echo "usage: $0 ${__usage_progname} <volume> [<peer>[:</path/to/brick>]]";;
	(gfid2path)
		echo "usage: $0 ${__usage_progname} <volume> <gfid>";;
	(path2gfid)
		echo "usage: $0 ${__usage_progname} </path/to/file>";;
	(*) sed -e 's/^	//' <<END_OF_USAGE
	usage: $0 <command> [args]
	commands:
	  status		Report health status of a volume.
	  list			List files on the local peer which need fixing.
	  remove		Remove a file from all bricks in a volume.
	  compare		Compare files between peers.
	  gfid2path		Find the file(s) for a given gfid.
	  path2gfid		Find the gfid for the given file(s).
	  help			Display help about a command.

	Report bugs at http://github.com/major0/glusterfs-recovery/issues
END_OF_USAGE
	;;
	esac

	unset __usage_progname
	exit 0
}

## volume_is_valid
# @1 volume name
volume_is_valid() { gluster volume list|grep -q "^${1}$"; }

## volume_is_arbiter
# @1 volume name
# @2 hostname
volume_is_arbiter() { gluster volume info "${1}" | grep -q "^Brick.*${2}.*(arbiter)"; }

## getpeer
# @1 volume name
# Report the first suitable peer for a given volume
getpeer()
{(
	: "getpeer($*)"
	__getpeer_volume="${1}"
	gluster volume info "${__getpeer_volume}"|while read __getpeer_line; do
		set -- ${__getpeer_line}
		case "${1}" in
		(Bricks*|Brick*\(arbiter\)*) continue;;
		(Brick*)
			set -- "${2%%:*}" "${2#*:}"
			if test "${1}" != "${HOSTNAME}"; then
				echo "${1}"
				exit 0
			fi
			continue
			;;
		esac
	done
	exit 0
)}

## list_peers
# @1 volume name
# Report all peers of a given volume
list_peers()
{(
	: "list_peers($*)"
	__list_peers_volume="${1}"
	gluster volume info "${__list_peers_volume}"|while read __list_peers_line; do
		set -- ${__list_peers_line}
		case "${1}" in
		(Brick*) echo "${2%%:*}"; continue ;;
		esac
	done
)}

## brick_path
# @1 volume name
# @2 peer
# Find the brick path for a given peer
brick_path()
{(
	: "brick_path($*)"
	__brick_path_volume="${1}"
	__brick_path_peer="${2}"
	gluster volume status "${__brick_path_volume}"|while read __brick_path_line; do
		set -- ${__brick_path_line}
		case "${1}" in
		(Brick*)
			set -- "${2%%:*}" "${2#*:}"
			if test "${1}" = "${__brick_path_peer}"; then
				echo "${2#*:}"
				exit 0
			fi
			continue
			;;
		(Task*)	exit 1;;
		esac
	done
)}

## inum2path
# @1 brick path
# @2 inum
inum2path() {
	: "inum2path($*)"
	find "${1}" -inum "${2}" ! -path '*/.glusterfs/*'| \
		while read inum2path_line; do
			printf '%s\n' "${inum2path_line#${1}}"
		done
}

## gfid2path
# @1 brick path
# @2 gfid
#
# The .glusterfs directory in the brick root has files named by GFIDs.
#   IF: The GFID identifies a directory, then this file is a symlink to the actual
#       directory.
#   IF: The GFID identifies a file, then this file is a hard-link to the actual file.
gfid2path()
{
	: "gfid2path($*)"
	set -- "${1}" "${2}" "${2%${2##????}}"
	set -- "${1}" "${2}" "${1}/.glusterfs/${3%??}/${3#??}"
	set -- "$@" "${3}/${2}"
	# @1 brick path
	# @2 gfid
	# @3 gfid_prefix
	# @4 gfid_path
	test -e "${4}" || die "no such file or directory '${4}'"

	if test -h "${4}"; then
		realpath "${3}/$(readlink "${4}")"
	else
		set -- "$@" "$(stat --printf='%i' "${4}")"
		# @5 inum
		inum2path "${1}"  "${5}"
	fi
}

## path2gfid
# @1 file path
#
# Gluster stores the gfid for every file as an extended attribute named
# 'trusted.gfid'
path2gfid()
{
	: "path2gfid($*)"
	set -- "$(getfattr -e hex -n trusted.gfid "${1}" 2>/dev/null|sed -e '/^#/d')"
	set -- "${1#trusted.gfid=0x}"
	set -- "${@}" "$(printf '%.8s' "${1}")"
	set -- "${@}" "$(printf '%.4s' "${1#${2}}")"
	set -- "${@}" "$(printf '%.4s' "${1#${2}${3}}")"
	set -- "${@}" "$(printf '%.4s' "${1#${2}${3}${4}}")"
	set -- "${@}" "${1#${2}${3}${4}${5}}"
	shift
	echo "${1}-${2}-${3}-${4}-${5}"
}

## volume_status
# @1 volume to examine
volume_status()
{
	: "volume_status($*)"
	__volume_status_vol="${1}"
	__volume_status_peer="${2}"
	gluster volume heal "${1}" info | while read __volume_status_line; do
		set -- ${__volume_status_line}
		: "volume_status: case($*)"
		case "${1}" in
		(Brick*)	__volume_status_peer="${2%%:*}"; continue ;;

		(Status*)	test "$#" -eq 2 || echo "${__volume_status_peer}=offline"; continue;;

		(Number*)	if isnum "${4}" && test "${4}" -gt '0'; then
					printf "${__volume_status_peer}=${4} "
				fi
				;;
		esac
	done
}

## heal_list
# @1 volume to examine
# @2 hostname to use for localhost
heal_list()
{
	: "heal_list($*)"
	__heal_list_volume="${1}"
	__heal_list_hostname="${2}"
	# pipes spawn a subshell
	export __heal_list_hostname

	gluster volume heal "${__heal_list_volume}" info | while read __heal_list_line; do
		test -n "${__heal_list_line}" || continue
		case "${__heal_list_line}" in
		(Brick*:*)
			: "__heal_list_line='$*'"
			set -- ${__heal_list_line}

			case "${2}" in
			(${__heal_list_hostname}:*)
				__heal_list_brick="${2##*:}";;
			(*)	__heal_list_brick=
				continue;;
			esac

			test -d "${__heal_list_brick}" || die "local brick not found '${__heal_list_brick}'"
			continue
			;;
		esac

		test -n "${__heal_list_brick}" || continue

		case "${__heal_list_line}" in
		(*gfid:*)
			__heal_list_head="${__heal_list_line##*:}"
			__heal_list_tail="${__heal_list_head#*/}"
			if test "${__heal_list_head}" != "${__heal_list_tail}"; then
				__heal_list_head="${__heal_list_head%${__heal_list_tail}}/${__heal_list_tail}"
			else
				__heal_list_head="${__heal_list_head%>}"
			fi
			gfid2path "${__heal_list_brick}" "${__heal_list_head}"
			;;

		(Status:*|Number*of*entries:*)
			continue;;

		(*)
			# note: there appears to be a bug in the gluster tools
			# which appends whitespace to the file name
			printf '%s\n' "${__heal_list_line%% }";;
		esac
	done
	unset __heal_list_volume
	unset __heal_list_hostname
}

## file_cmp
# @1 /path/to/local/brick
# @2 host:/path/to/remote/brick
# @3 optional /path/to/backup
file_cmp()
{
	: "file_cmp($*)"
	__file_cmp_local_path="${1}"
	__file_cmp_remote_host="${2%%:*}"
	__file_cmp_remote_path="${2#*:}"
	__file_cmp_backup_path="${3}"

	while read __file_cmp_line; do
		__file_cmp_backup_size=0
		__file_cmp_remote_size=0
		__file_cmp_local_size=0
		if test -e "${__file_cmp_local_path}${__file_cmp_line}"; then
			__file_cmp_local_size="$(stat --printf='%s' "${__file_cmp_local_path}${__file_cmp_line}" 2>/dev/null)"
		fi
		if test -e "${__file_cmp_backup_path}${__file_cmp_line}"; then
			__file_cmp_backup_size="$(stat --printf='%s' "${__file_cmp_backup_path}${__file_cmp_line}")"
		fi
		if rtest "${__file_cmp_remote_host}" -e "\"${__file_cmp_remote_path}${__file_cmp_line}\""; then
			__file_cmp_remote_size="$(rstat "${__file_cmp_remote_host}" --printf='%s' "\"${__file_cmp_remote_path}${__file_cmp_line}\"")"
		fi
		if test "${__file_cmp_local_size}" -eq "${__file_cmp_remote_size}"; then
			if test "${__file_cmp_local_size}" -gt '0'; then
				echo "OK:${__file_cmp_line}"
			else
				echo "!:${__file_cmp_line}"
				continue
			fi
		elif test "${__file_cmp_local_size}" -gt "${__file_cmp_remote_size}"; then
			printf '%d>%d:%s\n' "${__file_cmp_local_size}" "${__file_cmp_remote_size}" "${__file_cmp_line}"
		elif test "${__file_cmp_backup_size}" -gt "${__file_cmp_remote_size}"; then
			printf '%d>%d:%s\n' "${__file_cmp_local_size}" "${__file_cmp_remote_size}" "${__file_cmp_line}"
		else
			printf '%d<%d:%s\n' "${__file_cmp_local_size}" "${__file_cmp_remote_size}" "${__file_cmp_line}"
		fi
	done
	unset __file_cmp_backup_size
	unset __file_cmp_remote_size
	unset __file_cmp_local_size
	unset __file_cmp_local_path
	unset __file_cmp_remote_host
	unset __file_cmp_remote_path
	unset __file_cmp_backup_path
}

## rtest
# @1 host
# ... args to remote test
rtest()
{
	: "rtest($*)"
	__rtest_remote_host="${1}"
	shift
	set -- $(ssh -n -T -o PasswordAuthentication=no "${__rtest_remote_host}" "STRING_ERR=\"\$(test $(for arg;do printf "'${arg}' ";done) 2>&1 1>/dev/null)\";echo \"\$? \${STRING_ERR}\"")
	__rtest_return="${1}"
	shift 1
	test "$#" -eq 0 || echo "$*" >&2
	set -- ${__rtest_return}
	unset __rtest_remote_host
	unset __rtest_return
	return $1
}

## rstat
# @1 host
# ... args to remote stat
rstat()
{
	: "rstat($*)"
	__rstat_remote_host="${1}"
	shift
	set -- $(ssh -n -T -o PasswordAuthentication=no "${__rstat_remote_host}" "STRING_OUT=\"\$(stat $(for arg;do printf "'${arg}' ";done) 2>/dev/null)\";printf \"\$? \${STRING_OUT}\"")
	__rstat_return="${1}"
	shift 1
	echo "$*"
	set -- ${__rstat_return}
	unset __rstat_remote_host
	unset __rstat_return
	return $1
}

## Argument Processing
while getopts ':h' __arg; do
	case "${__arg}" in
	(h)	__usage "${0}";;

	# Missing optional arguments
	(:)	case "${OPTARG}" in
		(*)	__usage "${0}" "option '-${OPTARG}' requires an argument";;
		esac;;

	## Long Argument handling
	# POSIX: Each time it is invoked, the getopts utility shall
	# place the value of the next option in the shell variable
	# specified by the 'name' operand and the index of the next
	# argument to be processed in the shell variable OPTIND.
	#
	# NOTE - During error handling Bash and Zsh "appear" to point
	# OPTIND at the argument that had the error, a behavior not
	# decribed by POSIX.
	(\?)	: "extended options: index=${OPTIND}, arg='${__arg}', optarg='${OPTARG}'"
		shift $(__shiftarg)
		: "extended arg: '${1}'"

		case "${1}" in
		(--help)
			OPTARG="$(__optarg "${1}" "${2}")"
			test -z "${OPTARG}" || __usagusage "${OPTARG}"
			__usage "${0}"; exit 0;;

		##
		# Our standard argument handling
		(--)	OPTIND="$((${OPTIND} + 1))"; break;;
		(-*)	__usage "${0}" "unknown option '${1}'"
			exit 1;;
		(*)	break;;
		esac

		shift ${OPTIND}
		OPTIND=1;;
	esac
done
unset __arg
shift $((${OPTIND} - 1))
OPTIND=1
unset OPTARG

test -n "${HOSTNAME}" || HOSTNAME="$(hostname)"
export HOSTNAME

test "$#" -gt '0' || __usage "${0}" 'no command specified'
COMMAND="${1}";shift
for arg; do case "${arg}" in (-h|--help) __usage "${COMMAND}";; esac; done

## Main
case "${COMMAND}" in
(status)
	STATUS_LIMIT=
	STATUS_TOTAL=
	while getopts ':t:l:' __status_arg; do
		case "${__status_arg}" in
		(l)	STATUS_LIMIT="${OPTARG}";;
		(t)	STATUS_TOTAL="${OPTARG}";;
		(:) # Missing optional arguments
			case "${OPTARG}" in
			(*)	__usage "${COMMAND}" "option '-${OPTARG}' requires an argument";;
			esac;;

		## Long Argument handling
		# POSIX: Each time it is invoked, the getopts utility shall
		# place the value of the next option in the shell variable
		# specified by the 'name' operand and the index of the next
		# argument to be processed in the shell variable OPTIND.
		#
		# NOTE - During error handling Bash and Zsh "appear" to point
		# OPTIND at the argument that had the error, a behavior not
		# decribed by POSIX.
		(\?)	: "extended options: index=${OPTIND}, arg='${__status_arg}', optarg='${OPTARG}'"
			shift $(__shiftarg)
			: "extended arg: '${1}'"

			case "${1}" in
			(--limit|--limit=*)
				STATUS_LIMIT="$(__optarg "${1}" "${2}")" ;;

			(--total|--total=*)
				STATUS_TOTAL="$(__optarg "${1}" "${2}")" ;;

			##
			# Our standard argument handling
			(--)	OPTIND="$((${OPTIND} + 1))"; break;;
			(-*)	__usage "${COMMAND}" "unknown option '${1}'"
				exit 1;;
			(*)	break;;
			esac

			shift ${OPTIND}
			OPTIND=1;;
		esac
	done
	unset __status_arg
	shift $((${OPTIND} - 1))
	OPTIND=1
	unset OPTARG

	STATUS_LIMIT="${STATUS_LIMIT:=0}"
	isnum "${STATUS_LIMIT}" || __usage "${COMMAND}" "invalid limit specified '${STATUS_LIMIT}'"

	STATUS_TOTAL="${STATUS_TOTAL:=${STATUS_LIMIT}}"
	isnum "${STATUS_TOTAL}" || __usage "${COMMAND}" "invalid total limit specified '${STATUS_TOTAL}'"

	gluster volume list > /dev/null 2>&1 || die "'gluster' command unavailable"
	test "$#" -gt '0' || set -- $(gluster volume list)
	for VOLUME; do volume_is_valid "${VOLUME}" || die "invalid volume '${VOLUME}'";done
	VOLUME_RET=0
	for VOLUME; do
		VOLUME_ERROR=0
		VOLUME_TOTAL=0
		VOLUME_REPORT=
		VOLUME_STATUS="$(volume_status "${VOLUME}")"
		if test -z "${VOLUME_STATUS}"; then
			echo "${VOLUME}: OK"
			continue
		fi

		for ENTRY in ${VOLUME_STATUS} __DONE__; do
			case "${ENTRY}" in
			(__DONE__)
				if test "$#" -gt '1'; then
					VOLUME_REPORT="${VOLUME}=${VOLUME_TOTAL}, ${VOLUME_REPORT}"
				else
					VOLUME_REPORT="total=${VOLUME_TOTAL}, ${VOLUME_REPORT}"
				fi
				if test "${VOLUME_RET}" -gt '0'; then
					echo "ERROR: ${VOLUME_REPORT}"
				else
					echo "${VOLUME}: OK"
				fi
				break;;

			(*)	if ! test -z "${VOLUME_REPORT}"; then
					VOLUME_REPORT="${VOLUME_REPORT}, "
				fi

				ENTRY_VALUE="${ENTRY##*=}"
				if isnum "${ENTRY_VALUE}"; then
					test "${ENTRY_VALUE}" -lt "${VOLUME_ERROR}" || VOLUME_ERROR="${ENTRY_VALUE}"
					VOLUME_TOTAL="$((${VOLUME_TOTAL} + ${ENTRY_VALUE}))"
					if test "${VOLUME_TOTAL}" -gt "${STATUS_TOTAL}" || \
					   test "${VOLUME_ERROR}" -gt "${STATUS_LIMIT}"; then
						VOLUME_RET=1
					fi
				else
					# brick is offline?
					VOLUME_RET=1
				fi
				VOLUME_REPORT="${VOLUME_REPORT}${ENTRY}"
				;;
			esac
		done
	done
	exit ${VOLUME_RET}
	;;

(list|ls)
	gluster volume list > /dev/null 2>&1 || die "'gluster' command unavailable"
	test "$#" -gt 0 || __usage "${COMMAND}" 'no volume specified'
	for VOLUME; do volume_is_valid "${VOLUME}" || die "invalid volume '${VOLUME}'";done
	for VOLUME; do heal_list "${VOLUME}" "${HOSTNAME}"; done
	;;

(remove|rm)
	RECURSE=false
	FORCE=false
	DRY_RUN=false
	INDEX=0
	for arg; do
		case "${arg}" in
		(-r|--recurse)	RECURSE=true;;
		(-f|--force)	FORCE=true;;
		(-n|--dry-run)	DRY_RUN=true;;
		(*)		set -- "$@" "${arg}";;
		esac
		INDEX="$((${INDEX}+1))"
	done
	shift "${INDEX}"
	: "remove: ${@}"

	gluster volume list > /dev/null 2>&1 || die "'gluster' command unavailable"
	test "$#" -gt '0' || __usage "${COMMAND}" 'no volume specified'
	volume_is_valid "${1}" || die "invalid volume '${1}'"
	VOLUME="${1}"
	shift

	if ! LOCAL_BRICK="$(brick_path "${VOLUME}" "${HOSTNAME}")"; then
		die "local peer not a member of volume '${VOLUME}'"
	fi

	test "$#" -ge '1' || __usage "${COMMAND}" 'no file specified'

	if test -e "${1}"; then
		FILE="$(__realpath "${1}")"
	else
		FILE="$(__realpath "${LOCAL_BRICK}${1}")"
	fi
	FILE="${FILE#${LOCAL_BRICK}}"
	test -e "${LOCAL_BRICK}${FILE}" || die "file or directory not in volume '${VOLUME}'"

	if test -d "${LOCAL_BRICK}${FILE}"; then
		if ! ${RECURSE}; then
			warn "The specified path is a directory. Out-of-band removal of an entire"
			warn "directory tree is generally a bad idea."
			warn "Specify '--recurse' to remove the entire directory and all warn the"
			warn "files within the tree."
			exit 1
		fi
		RECURSE='-rf'
	else
		RECURSE=
	fi

	GFID="$(path2gfid "${LOCAL_BRICK}/${FILE}")"
	GFID_PATH="${GFID%${GFID##????}}"
	GFID_PATH="/.glusterfs/${GFID_PATH%??}/${GFID_PATH#??}/${GFID}"
	test -e "${LOCAL_BRICK}${GFID_PATH}" || die "cannot find gfid path for file '${FILE}'"

	for PEER in $(list_peers "${VOLUME}"); do
		if ! ssh -n -T -o PasswordAuthentication=no "${PEER}" exit 0; then
			die "localhost not authorized to ssh to '${PEER}'"
		fi
	done

	for PEER in $(list_peers "${VOLUME}"); do
		REMOTE_BRICK="$(brick_path "${VOLUME}" "${PEER}")"

		if test "${PEER}" = "${HOSTNAME}"; then
			if ! test -e "${REMOTE_BRICK}${FILE}"; then
				warn "file does not exist on ${PEER}"
			else
				set -- "${REMOTE_BRICK}${FILE}"
			fi

			if ! test -e "${REMOTE_BRICK}${GFID_PATH}"; then
				warn "gfid path does not exist on ${PEER}"
				continue
			else
				set -- "$@" "${REMOTE_BRICK}${GFID_PATH}"
			fi

			if ${DRY_RUN}; then
				echo rm ${RECURSE} "'${REMOTE_BRICK}${FILE}'" "'${REMOTE_BRICK}${GFID_PATH}'"
			elif ${FORCE}; then
				rm ${RECURSE} "${REMOTE_BRICK}${FILE}" "${REMOTE_BRICK}${GFID_PATH}"
			fi

		else
			if ! rtest "${PEER}" -e "${REMOTE_BRICK}${FILE}"; then
				warn "file does not exist on ${PEER}"
			else
				set -- "${REMOTE_BRICK}${FILE}"
			fi

			if ! rtest "${PEER}" -e "${REMOTE_BRICK}${GFID_PATH}"; then
				warn "gfid path does not exist on ${PEER}"
				continue
			else
				set -- "$@" "${REMOTE_BRICK}${GFID_PATH}"
			fi

			if ${DRY_RUN}; then
				echo ssh -n -T -o PasswordAuthentication=no "${PEER}" rm ${RECURSE} "'${REMOTE_BRICK}${FILE}'" "'${REMOTE_BRICK}${GFID_PATH}'"
			elif ${FORCE}; then
				ssh -n -T -o PasswordAuthentication=no "${PEER}" rm ${RECURSE} "${REMOTE_BRICK}${FILE}" "${REMOTE_BRICK}${GFID_PATH}"
			fi
		fi
	done
	if ! "${FORCE}" && ! "${DRY_RUN}"; then
		warn "This command will permanently remove data from your filesystem and"
		warn "should only be used as a tool of last resort and ONLY after warn"
		warn "backups have been made of ALL bricks in the volume."
		warn "Use '--force' to permanantely remove these files."
	fi
	;;

(compare|cmp)
	gluster volume list > /dev/null 2>&1 || die "'gluster' command unavailable"
	test "$#" -gt 0 || __usage "${COMMAND}" 'no volume specified'
	volume_is_valid "${1}" || die "invalid volume '${1}'"
	if ! gluster volume info "${1}"|grep -iq '^type:.*replicate'; then
		die 'compare only valid on replicated volumes'
	fi
	VOLUME="${1}"
	shift

	if volume_is_arbiter "${VOLUME}" "${HOSTNAME}"; then
		die 'can not run compare from arbiter'
	fi

	if ! LOCAL_PREFIX="$(brick_path "${VOLUME}" "${HOSTNAME}")"; then
		die "local peer not a member of volume '${VOLUME}'"
	fi

	if test "$#" -gt '0'; then
		if ! ssh -n -T -o PasswordAuthentication=no "${1%%:*}" exit 0; then
			die "localhost not authorized to ssh to '${1%%:*}'"
		fi
		REMOTE_HOST="${1%%:*}"

		if volume_is_arbiter "${VOLUME}" "${REMOTE_HOST}"; then
			die 'can not compare against an arbiter'
		elif test "${1}" != "${REMOTE_HOST}"; then
			if ! rtest "${REMOTE_HOST}" -e "${1#*:}"; then
				die "invalid remote path '${1#*:}'"
			fi
			REMOTE_PREFIX="${1#*:}"
		elif ! REMOTE_PREFIX="$(brick_path "${VOLUME}" "${REMOTE_HOST}")"; then
			die "peer '${REMOTE_HOST}' not a member of volume '${VOLUME}'"
		fi
		shift
	else
		if ! REMOTE_HOST="$(getpeer "${VOLUME}")"; then
			die "no available peers for volume '${VOLUME}'"
		fi

		if ! ssh -n -T -o PasswordAuthentication=no "${REMOTE_HOST}" exit 0; then
			die "localhost not authorized to ssh to '${REMOTE_HOST}'"
		fi
	fi

	heal_list "${VOLUME}" "${HOSTNAME}" | file_cmp "${LOCAL_PREFIX}" "${REMOTE_HOST}:${REMOTE_PREFIX}" ${BACKUP_PREFIX}
	;;

(gfid2path)
	gluster volume list > /dev/null 2>&1 || die "'gluster' command unavailable"
	test "$#" -gt '0' || __usage "${COMMAND}" 'no volume specified'
	volume_is_valid "${1}" || die "invalid volume '${1}'"
	VOLUME="${1}"
	shift

	if ! BRICK="$(brick_path "${VOLUME}" "${HOSTNAME}")"; then
		die "localhost not a member of volume '${VOLUME}'"
	fi

	test "$#" -ge '1' || __usage "${COMMAND}" 'no gfid specified'
	test "$#" -le '1' || __usage "${COMMAND}" 'too many arguments'
	gfid2path "${BRICK}" "${1}"
	;;

(path2gfid)
	test "$#" -ge '1' || __usage "${COMMAND}" 'no file specified'
	test "$#" -le '1' || __usage "${COMMAND}" 'too many arguments'
	test -e "${1}" || __usage "${COMMAND}" "no such file '${1}'"
	path2gfid "${1}"
	;;

(help)  test "$#" -eq '0' || __usage "${1}"
	__usage "${0}" ;;
(*)	__usage "unknown command '${COMMAND}'";;
esac