#!/bin/bash
#
# chkconfig: 345 70 8
# description: Loads and unloads the drbd module
#
# complete rewrite from scratch by Lars Ellenberg <l.g.e@web.de> in June 2002
# for the drbd project
# reworked the signal handling April 2003
# based on discussions with Joachim Banzhaf <jbanzhaf@ngi.de>
# released under the same terms as drbd itself,
# or the GPL, whatever suits you best.
#
# for questions wrt the functionality search for QQQ in the comments below
# for style issues see end of file
#
### BEGIN INIT INFO
# Provides: drbd
# Required-Start: $network $syslog
# Required-Stop:
# Default-Start:  3 5
# Default-Stop:   0 1 2 6
# Description:    Control drbd resources.
### END INIT INFO
#
# some notes: {{{2
# if you use vim, press zi now, then try zr and zm too ... :)
#
# QQQ Required-Start: $sshd
#
# QQQ Provides: datadisk ?
#
# QQQ should there be an auto background mode as to
#     not block other init scripts?
# AAA It is there!
#	  see skip-wait, and if needed, (-)inittimeout
#     or load-only
#
# Q   I get spurious "synchronization finished" messages
# A   No problem. If the nodes find each other very quickly,
#     and there is no need for synchronization, it may well
#     happen, that these "do nothing" childs get "reaped".
#     Thus the message.
#
# Q   Will you please clean up this ugly signal passing code?
# A   Sure. Eventually. I do not like it either, but it seems
#     to work most of the time ;)
#     In drbd 0.7 we will hopefully have more straight forward
#     resource scripts.
#
# functionality translated from drbd.pl CVS 1.5 to
# BASH_VERSINFO=(2 05 0 1 release i386-suse-linux)
# should now work with
# BASH_VERSINFO=(2 02 1 1 release i686-pc-linux-gnu)
# and better, too
# !! 2.05b has changed wait builtin semantics, at least for me.
# !! wait now is interupted by signals.
# altered some small things, it does now work again with
# BASH_VERSINFO=(2 05b 0 1 release i586-suse-linux)
#
# parser is more permissive wrt whitespace
# easily extended for new options or more than two partners
#
# shouldn't there be more descriptive comments? ;)
#
CVSID='$Id: drbd,v 1.55 2004/03/04 07:38:18 lars Exp $'
CONTACT='Lars Ellenberg <l.g.e@web.de>, drbd-user@lists.linbit.com'

##################################################}}}1
#
# Environment and shell options
#
##################################################{{{1

# If I was confident enough, I would use set -e for the whole script
# so each failed simple command would fail the script.
# I had to use error checks on all simple commands ...
# for now only set this for the startup section, so I can give less
# missleading error messages

trap "echo Error during script startup. This may be an incompatible bash version" EXIT
set -e
# check for required bash features {{{2

if ! [ "${BASH_VERSINFO[0]}" -ge 2 -a "${BASH_VERSINFO[1]//[^0-9]/}" -ge 2 ] ; then
	echo "bash version is older than known to work."
	echo "  If it works despite of that, report as known good version to"
	echo "  $CONTACT"
	echo "  so this value can be adjusted in the script."
	echo "  Otherwise: please upgrade to a newer bash version. -- Thanks"
fi

set -o noglob            # disable expansion of *? [] etc
set -o noclobber         # don't overwrite files with '>' redirection

shopt -s extglob         # enable extended pattern matching features
shopt -s expand_aliases  # enable aliases

# test compatibility early
try1=try2
try2="something"
if [ ${!try1} != "something" ] ; then
	 echo "Indirection operator not implemented."
	 echo "This script will not work with this version of bash."
	 exit -1
fi
unset try1 try2
case yes713 in
@(no|yes)+([0-9])) ;;
*)
	 echo "Extended pattern matching does not work."
	 echo "This script will not work with this version of bash."
	 exit -1
	 ;;
esac
# [[ dummy ]] # will fail the script if not implemented due to set -e above
# (( 1 ))     # dito

# some default [paranoia] settings {{{2

: ${VERBOSITY:=-1}
IFS=$' \t\n' # just in case
PATH=/sbin:/usr/sbin:/bin:/usr/bin

# short hostname (without domain)
# since /bin/hostname might hang in some cases,
# and $HOSTNAME might be set different,
# use procfs directly.

# btw, bash-2.02.1(1) seems to have problems reading this
# this hack would make it work, though...
# read HOST < <(cat /proc/sys/kernel/hostname)
read HOST < /proc/sys/kernel/hostname
HOST=${HOST%%.*}

# since the location is not guaranteed to be the same on all distibutions,
# do not use full path
MODPROBE="modprobe"
RMMOD="rmmod"
UMOUNT="umount -v"
MOUNT="mount -v"
FUSER="fuser"
TOUCH="touch"

DRBD_SETUP="drbdsetup"
PROC_DRBD="/proc/drbd"
CONFIG="/etc/drbd.conf"
CONFIG_PARSED="/var/lib/drbd/drbd.conf.parsed"

PING="ping -c 1 -w 3"
# do we have [-w deadline] option?
$PING localhost &> /dev/null || {
	[ $? -eq 2 ] && PING="timeout_exec -3 ping -c 1"
}

base=${0##*/}
link=${base#*[SK][0-9][0-9]}
link=${link%.sh}
link=${link#rc}

type -P logger &>/dev/null && HAVE_LOGGER=true || HAVE_LOGGER=false

if $HAVE_LOGGER ; then
	LOGECHO="logger -t "$link" -s -p user.notice"
	LOGERR="logger -t "$link" -s -p user.err"
else
	LOGECHO="echo $link:"
	LOGERR="echo $link:"
fi
USLEEP="sleep 1"

# QQQ thought this could be useful
# TERMINAL=/dev/tty
# maybe we can use $CONSOLE, which should be exported by init anyways?
#
for TERMINAL in /proc/$$/fd/0 /dev/tty /dev/console ""
do
   [ -z "$TERMINAL" ] && break
   ( [ -t 1 ] < $TERMINAL &> $TERMINAL ) 2>/dev/null && break
done
if [ -z "$TERMINAL" ] ; then
	$LOGERR "FATAL: no terminal"
	exit -1
fi


# frame work for commandline options part I {{{2
# I use getopt, if it is in PATH. if getopt is not found, it will
# (hopefully) silently work anyways, though the user has to supply correct
# ordering and may not abbreviate
OPTIONS='dry-run,info,parse,dummy,config:,dontwait,dontkill'
GETOPT="`type -p getopt` -s bash -n $link -a -l $OPTIONS -- . " \
|| GETOPT="printf '%q '"

##################################################}}}1
#
# "compatibility" functions for bash versions which do not know yet about
# [[ ]], (( )) and the like
#
##################################################{{{1

mymatch() # {{{2
# mymatch <something> <pattern>
# returns 0 for true, -1 for false.
# replaces [[ something == pattern ]]
{
	case "$1" in $2) return 0;; *) return -1;; esac
}

##################################################}}}1
#
# Configuration Template
#
##################################################{{{1

# defines a /complete/ CONF structure
#
# first word of template value:
#   __optional__   : empty default values , not used right now
#   __required__   : required values
#   {yes,no}       : toggles
#   __appended__   : values to be concatenated
#   __implicit__   : implicitly defined through internal data
#                  : or by use of __appended__
#   any other value: default value if not mentioned in config file
#           !!!    : may NOT contain whitespace
#
# second to last word of template value are interpreted as extended bash
# pattern, which the actual TOKEN_LINE must match.
# special patterns are -x and -b, which test for executable, or
# block device, respectively
#
# if an option appears without explicit = in the config file, and
#    default is *not* {yes,no}
#         -> Parse error
#    default *is* {yes,no}
#         -> assigned the "not default" value
#         this makes probably only sense with boolean toggles
#         like skip-wait
#
# if something is in the CONF file but not listed here it is
# treated as error
#
# "HOST_*" is special:
# it is replaced by the actual section hostname

declare -a TEMPLATE # {{{2
TEMPLATE=( \
	__hash__NNN                                                           \
	                                                                      \
	skip-wait        "no    skip-wait?(=@(yes|no|true|false|1|0|on|off))" \
	load-only        "no    load-only?(=@(yes|no|true|false|1|0|on|off))" \
	inittimeout      "-0               *=?(-)+([0-9])          "          \
	protocol         "__required__     *=@(a|A|b|B|c|C)        "          \
	fsckcmd          "__required__     -x                      "          \
	incon-degr-cmd   "0                -x                      "          \
	                                                                      \
	disk              __implicit__                                        \
	disk:disk-size   "__appended__     +([^ ])=+([0-9])?([kMG])"          \
	disk:do-panic    "__appended__     do-panic                "          \
	                                                                      \
	net               __implicit__                                        \
	net:connect-int  "__appended__     +([^ ])=+([0-9])        "          \
	net:ping-int     "__appended__     +([^ ])=+([0-9])        "          \
	net:skip-sync    "__appended__     skip-sync?(=[12])       "          \
	net:sync-rate    "__appended__     +([^ ])=+([0-9])?([kMG])"          \
	net:sync-min     "__appended__     +([^ ])=+([0-9])?([kMG])"          \
	net:sync-max     "__appended__     +([^ ])=+([0-9])?([kMG])"          \
	net:sync-nice    "__appended__     +([^ ])=+([0-9\+\-])    "          \
	net:sync-group   "__appended__     +([^ ])=+([0-9])        "          \
	net:timeout      "__appended__     +([^ ])=+([0-9])        "          \
	net:ko-count     "__appended__     +([^ ])=+([0-9])        "          \
	net:tl-size      "__appended__     +([^ ])=+([0-9])        "          \
	net:sndbuf-size  "__appended__     +([^ ])=+([0-9])?([kMG])"          \
	                                                                      \
	HOSTS             __implicit__                                        \
	                                                                      \
	HOST_device      "__required__     *=/dev/nb*              "          \
	HOST_disk        "__required__     -b                      "          \
	HOST_port        "__required__     *=+([0-9])              "          \
	HOST_address     "__required__     *=+([_A-Za-z0-9.-])"               \
)
	# QQQ restrict this to ip addresses in dotted quad notation.
	# otherwise the sanity check below would not be reliable.
	# HOST_address  "__required__     *=+([0-9])\.+([0-9])\.+([0-9])\.+([0-9])"
TEMPLATE="__hash__${#TEMPLATE[*]}"
readonly TEMPLATE

# this will store the parsed configuration data
CONF=(__hash__0)

##################################################}}}1
#
# Convenience Functions and Aliases, Logging
#
##################################################{{{1

# work around a bug in bash's LINENO
# MIND YOU! you have to include die's in braces, if you use it like
# do something || { die errtxt; }
alias die='reset_lineno; die_at "$0" "$FUNCNAME" "$LINENO"'
alias trace='debug 3 "%-50s # %s" "$FUNCNAME($*)"'
alias trace4='debug 4 "%-50s # %s" "$FUNCNAME($*)"'
alias trace0='debug 0 "%-50s # %s" "$FUNCNAME($*)"'

print_usage_and_exit() # {{{2
{
	case $link in

	datadisk)
		echo -e "\nUSAGE: datadisk [resource] start|stop|status"
		;;

	drbd)
		echo -e "\nUSAGE: drbd [resource] start|stop|status|reconnect|checkconfig"
		;;

		*)        cat <<-___
			Don't know what to do.
			I expect to be called with a script name of either datadisk or drbd.
			USAGE:
			drbd     [resource] start|stop|status|reconnect|checkconfig
			datadisk [resource] start|stop|status
			___
	esac
	OPTIONS=${OPTIONS//:/=arg}
	echo -e "OPTIONS: --${OPTIONS//,/ --}\n"
	exit 255
}

error() # {{{2
# to syslog with *.err, to stderr, AND to stdout (unless stdout == stderr)
{
	$LOGERR "ERROR: $*" >&2
	[ /dev/stderr -ef /dev/stdout ] || echo "$link: ERROR: $*"
}

note() # {{{2
# to syslog with *.notice
# to syslog with *.err, to stderr, AND to stdout (unless stdout == stderr)
{
	$LOGECHO "$*" >&2
	[ /dev/stderr -ef /dev/stdout ] || echo "$link: $*"
}

debug() # {{{2
# debug <lev> format arguments: printf > stderr
# level 1 is typically external (drbdsetup) command execution
# so copy everything of debuglevel 1 to syslog, too
{
	local lev=$1 tmp=$2 s=-s; shift 2
	tmp=$(printf -- "$tmp\n" "$@")
	if [ "$lev" -eq 1 ] && $HAVE_LOGGER ; then
		logger -t "$link" -p user.notice "$tmp"
	fi
	[ "$lev" -le "$VERBOSITY" ] || return 0
	echo "<$lev> $tmp" >&2
}

# LINENO hack {{{2
MAGIC_LINENO=$(( LINENO + 2 ))
reset_lineno()
{
	# "this is a dirty hack!"
	[ $LINENO != 2 ] && MAGIC_LINENO=0 # LINENO works! this is >=2.05b
	return 0
}

have_FUNCNAME() { [ "$FUNCNAME" = "have_FUNCNAME" ] || FUNCNAME="[bash is old]" ; }
have_FUNCNAME # bash <=2.02 does not have $FUNCNAME

die_at() # {{{2
# I like the perl way of saying "do_it or die"
#
# bash seems to have a bug regarding the internal variable LINENO
# it does not work in functions which call other functions.
# my workaround is the call to reset_lineno in the alias above,
# and add the known lineno of that function to the value here.
#
# update: seems to be fixed since 2.05b
# added some code to make it work with 2.02 up to 2.05b
#
{
	set +e
	local FILE=$1 FUNC=$2 LINE=$3 i
	shift 3

	# if logger is present, this goes to syslog and stderr
	# if not, this goes to stderr only
	echo
	if $HAVE_LOGGER ; then
		exec > >(logger -s -t drbd -p user.err) 
	else
		exec 1>&2
	fi

	# set some default message.
	[ -z "$*" ] && set -- "Died."

	# we are still reading the config?
	if [ -n "$CONFIG" ] ; then
		echo -e "\n  last token read from <$CONFIG> was"
		printf "%4i: %s%s\n%6s" "${TOKEN_LINE_NR:-0}" "$TOKEN" "$TOKEN_LINE" ""
		let i=0; while [ $i -lt ${#TOKEN} ] ; do let i=i+1; echo -n "^"; done
		echo
	fi

	# echo the message, i.e. the positional parameters.
	# should I use printf functionality?
	echo -ne "$*"

	# if we are in a function, append it to FILE
	# otherwise set MAGIC_LINENO to zero, since we are on the script level
	# and LINENO is valid.
	[ -n "$FUNC" ] && FILE="$FILE ($FUNC)" || MAGIC_LINENO=0
	[ $(( LINE + MAGIC_LINENO )) -lt ${MAIN_LINE_NO:-999999} ] || MAGIC_LINENO=0
	case "$*" in
	*$'\n')  :;;
	*)
		cat <<-___

		  stopped at $FILE line $(( LINE + MAGIC_LINENO )).

		  CVSID ${CVSID//\$/}
		  BASH_VERSINFO (${BASH_VERSINFO[*]})

		  If you feel this script is buggy, please report to
		  $CONTACT

		___
	esac
	# if it is a terminal, beep
	# [ -t 1 ] && printf "\a"
	exec 1>&- 2>&- # explicitly close stdout and stderr
	$USLEEP
	exit 255
}

##################################################}}}1
#
# "hash" emulation functions
#
##################################################{{{1
# not yet: delete
# "fetch" can be used like "exists"

# QQQ
# these "bash hashes" are kind of "proof of concept" only.
#
# all this could be done with variable indirection,
# and for many devices be much faster that way.
# I would need to map all non-identifier key names
# to underscore, then store would be
#   eval "${h}_${k//[^a-zA-Z0-9_]/_}=\$v"
# and fetch
#   x=${h}_${k//[^a-zA-Z0-9_]/_}; RET_VAL=(${!x})
# 
# should I rewrite this part?
#

store() # {{{2
# store [-new|-append] <Name> <Key> <Value>
# stores <Key> and <Value> in shell array <Name> on odd and even
# indices, respectively. If <Key> is already set, <Value> is
# reset, unless modifiers are given:
#	-new    : dies if <Key> already existed
#	-append : appends <Value> to the old Value
#	remember to include a separator, if you need one!
# return vale is always 0 ( dies on error )
# RET_VAL:
# -- falls through from fetch --
#	[0] = _old_ Value
#	[1] = index position of <Value> in shell array <Name>
#	[2] = number of <Key> <Value> pairs in the "hash"
#
# bash has arrays, not hashes, so since this loops over all
# elements of the array, it will soon get slow
#
{
	trace
	RET_VAL=()
	local must_be_new=false is_new=false append=false
	# modifier given?
	while mymatch "$1" "-*"; do
		case $1 in
		-new)
			must_be_new=true ;;
		-append)
			append=true ;;
		*)
			die "unknown modifier $1" ;;
		esac
		shift
	done

	local h=$1 k=$2 v=$3

	fetch "$h" "$k" || is_new=true
	local o=${RET_VAL[0]} i=${RET_VAL[1]} n=${RET_VAL[2]}

	if $is_new; then
		eval "$h[$((i - 1))]='$k'"
		eval "$h[0]='__hash__$((n + 2))'"
	else
		$must_be_new && { die "$PARSE: $k already set"; }
	fi

	if $append; then
		eval "$h[$i]='$o$v'"
	else
		eval "$h[$i]='$v'"
	fi
}

fetch() # {{{2
# fetch <Name> <Key>
# fetches <Value> for <Key> in shell array <Name>
# can also be used to test for existence of <Key>
#
# loops over the odd indices, if one matches <Key>, the
# corresponding even index <Value> is returned.
#
# RET_VAL:
#	[0] = Value
#	[1] = index position of <Value> in shell array <Name>
#	[2] = number <Key> <Value> elements in the "hash" array
#
# return value is 0 if <Key> was found, and
# otherwise 1
#
# in the latter case, [1] points to the next free indices if
# <Key> was not found (so [1] is greater than [2])
#
# bash has arrays, not hashes, so since this loops over all
# elements of the array, it will soon get slow
#
{
	trace4
	RET_VAL=()
	local h=$1 k=$2 i
	# did you know the reference indirection operator?
	local n=${!h}
	case $n in
		"" )
			eval "$h=(__hash__0)"; n=0 ;;

		__hash__+([0-9]) )
			n=${n#__hash__} ;;

		* )
			die "$h is not a valid hash" ;;
	esac
	eval "set -- \"\${$h[@]}\""
	shift
	let i=1; while [ $i -lt $# ] ; do
		[ "${!i}" == "$k" ] && break
		let i=i+2
	done
	let i=i+1
	RET_VAL=("${!i}" $i $n)
	debug 4 "$FUNCNAME returns (%s)" "${RET_VAL[*]}"
	return $(( i > n ))
}

Dump() # {{{2
# Dump <Name>
# pretty prints content of "hash" <Name>
# in a bash source-able form
{
	trace
	RET_VAL=()
	local h=$1 i
	local n=${!h}
	case $n in
		__hash__+([0-9]) )
			 n=${n#__hash__} ;;

		* )  die "$h is not a valid hash" ;;
	esac
	# could use the %q (quote) extension to printf,
	# but it looks too ugly
	# quote single quotes
	eval "set -- \"\${$h[@]//\'/\'\\\'\'}\""
	echo "$h=(\\"
	echo -e "\t$1 \\"
	let i=2; while [ $i -lt $# ] ; do
		printf "  %-30s" "'${!i}'"; let i=i+1
		echo " '${!i}' \\" ; let i=i+1
	done
	echo ")"
}

##################################################}}}1
#
# Parser
#
##################################################{{{1

# if line contains "=" token is line (leading and trailing whitespace removed)
# otherwise token is consecutive non-whitespace
# you may include as many whitespace as you like
# you may separate the "=" sign by whitespace

get_token() # {{{2
{
	# not local, but owned by parse_config
	TOKEN=''
	TOKEN_LINE=${TOKEN_LINE##+([$IFS])}
	until [ -n "$TOKEN_LINE" ] ; do
		read TOKEN_LINE
		[ $? == 0 -o -n "$TOKEN_LINE" ] || return 1
		let TOKEN_LINE_NR=TOKEN_LINE_NR+1
		if [ $VERBOSITY -lt 4 ] ; then
			echo -n .
			[ $((TOKEN_LINE_NR % 60)) == 0 ] && echo
		else
			debug 4 "- $CONFIG %4d: '%s'" $TOKEN_LINE_NR "$TOKEN_LINE"
		fi
		# strip comments. a "#" within a word is *not* considered comment
		# leader, it *has* to follow a whitespace or be the first character
		TOKEN_LINE=${TOKEN_LINE%%@(+([$IFS])|)#*}
	done
	case "$TOKEN_LINE" in
		*=*)
			# remove whitespace surrounding first equal sign
			TOKEN=${TOKEN_LINE/*([$IFS])=*([$IFS])/=}
			TOKEN_LINE=''
			;;
		*)
			TOKEN=${TOKEN_LINE%% *}
			TOKEN_LINE=${TOKEN_LINE#$TOKEN}
			;;
	esac
	debug 5 "%s" "- tok: '$TOKEN'" "rem: '$TOKEN_LINE'"
}

expect() # {{{2
# assertion: $TOKEN is $1. if not, die.
# $1 might be a pattern
{
	mymatch "$TOKEN" "$1" && return
	die "$PARSE: expected ($1), got '$TOKEN'"
}

skip_section() # {{{2
{
	expect "{"
	local -i depth=1
	until [ "$depth" == 0 ] ; do
		get_token || { die "$PARSE: unexpected end of file"; }
		case "$TOKEN" in
			"{") let depth=depth+1;;
			"}") let depth=depth-1;;
		esac
	done
}

check_token_type() # {{{2
# expects positional parameters to be the TEMPLATE value of the current TOKEN
{
	case $1 in

	__implicit__*)
		die "$PARSE: unknown option '${TOKEN%%=*}'" ;;

	__required__*)
		local hint=''
		[ -n "$TOKEN_LINE" ] && hint="\n\tdid you forget the = sign?"
		mymatch "$TOKEN" "*=*" || { die "$PARSE: need value for '$TOKEN'.$hint" ; }
		;;

	esac

	shift

	local pattern="$*" ret=0
	debug 5 "check_token_type: $TOKEN , $pattern"

	case $pattern in

	-x)
		pattern="executable"
		set -- ${TOKEN#*=}
		type -p $1 &> /dev/null ; ret=$? ;;

	-b)
		pattern="block device"
		test "$host" = "$HOST" || return 0
		test -b "${TOKEN#*=}" ; ret=$? ;;

	*)
		mymatch "$TOKEN" "$pattern"; ret=$?
		pattern="'$pattern'" ;;

	esac

	[ "$ret" != 0 ] && { die "bad value '$TOKEN',\n\t$pattern expected"; }
}

parse_option_section() # {{{2
{
	trace "for $RES"
	local option=$TOKEN key
	PARSE="$PARSE, $option"
	get_token ; expect "{"
	while get_token; do
		[ "$TOKEN" == "}" ] && break
		key=${TOKEN%%=*}
		fetch TEMPLATE "$option:$key"
		mymatch "$RET_VAL" "__appended__*" \
		|| { die "$PARSE: unknown option '$TOKEN'"; }
		check_token_type $RET_VAL
		store -append CONF "$RES:$option" " --$TOKEN"
	done
	[ "$TOKEN" == "}" ] || { die "$PARSE: unexpected end of file"; }
	PARSE=${PARSE%, $option}
}

parse_host_section() # {{{2
{
	trace "for $RES on $TOKEN"
	local host=$TOKEN
	PARSE="$PARSE, on $host"

	get_token ; expect "{"
	while get_token; do
		[ "$TOKEN" == "}" ] && break
		local key=${TOKEN%%=*}
		local val=${TOKEN##$key?(=)}
		fetch TEMPLATE "HOST_$key" \
		|| { die "$PARSE: unknown option '$key'"; }
		check_token_type $RET_VAL
		store -new CONF "$RES:on $host:$key" "$val"
	done
	[ "$TOKEN" == "}" ] || { die "$PARSE: unexpected end of file"; }
	store -append CONF "$RES:HOSTS" " $host "
	PARSE=${PARSE%, on $host}
}

parse_resource_section() # {{{2
# this does the main work
{
	trace0 "for $TOKEN"
	local RES=$TOKEN
	get_token; expect "{"
	while get_token; do
		case "$TOKEN" in

		"}")
			break
			;;

		disk|net)
			parse_option_section
			;;

		on)
			get_token
			parse_host_section
			;;

		*=*)
			local key=${TOKEN%%=*}
			local val=${TOKEN##$key=}
			if mymatch "$TOKEN" "HOST_*" || ! fetch TEMPLATE "$key" ; then
				die "$PARSE: unknown option '$key'"
			fi

			check_token_type $RET_VAL

			store -new CONF "$RES:$key" "$val"
			;;

		*)
			local key=$TOKEN
			fetch TEMPLATE "$key" \
			|| { die "$PARSE: unknown option '$key'"; }

			check_token_type $RET_VAL

			case $RET_VAL in

			yes\ *)
				store -new CONF "$RES:$TOKEN" no ;;

			no\ *)
				store -new CONF "$RES:$TOKEN" yes ;;

			esac
		esac
	done
	[ "$TOKEN" == "}" ] || { die "$PARSE: unexpected end of file"; }
	fetch CONF "$RES:HOSTS"
	local HOSTS=$RET_VAL
	mymatch "$HOSTS"  "* $HOST *" \
	|| { die "$PARSE: This host ($HOST) not mentioned."; }
	set -- $HOSTS
	[ $# -lt 2 ] \
	&& { die "$PARSE: No partner host mentioned."; }

	check_if_all_parameters_set
}

# parse_global_section() {{{2
_did_global_section=false
parse_global_section()
{
	trace0
	local RES=$TOKEN
	$_did_global_section && { die "you can have only one global section"$'\n'; }
	get_token; expect "{"
	while get_token; do
		case "$TOKEN" in

		"}")
			break
			;;

		# TODO: GLOBAL_TEMPLATE and GLOBAL_CONF hashes?
		# allow default configuration values, e.g. syncer?
		minor_count=+([0-9]))
			MINOR_COUNT=${TOKEN#minor_count=}
			;;

		disable_io_hints)
			MODULE_OPTIONS="disable_io_hints=1"
			;;

		*)
			die "invalid token in global section"$'\n'
			;;
		esac
	done
	[ "$TOKEN" == "}" ] || { die "$PARSE: unexpected end of file"; }
	_did_global_section=true
}

parse_config() # {{{2
# parse the configuration, and store this in source-able form
# on the next invocation we try to read this pre-parsed config
{
	trace0
	[ -z "$_SCRIPT_RECURSION" ] \
	&& echo "$link: pre-parsed needs update, parsing $CONFIG"

	if [ "$PROC_DRBD" == /proc/drbd -a "$CONFIG" == /drbd/drbd.conf ]
	then
	# If there are configured devices, and it was the newer CONFIG that
	# triggered the reread: warn
		local hot_devices
		hot_devices=$(sed '1d;2d;' < $PROC_DRBD | grep -v cs:\(Standalone\|Unconfigured\))
		if [ -e $CONFIG_PARSED ] && [ -n "$hot_devices" ] && [ $CONFIG -nt $CONFIG_PARSED ] ; then
			cat <<-_EOF

				$link: WARNING: change of configuration
				while drbd devices are configured!
				$hot_devices
			_EOF
			[ "$link" == datadisk ] && { die "use drbd restart first"$'\n'; }
			cat <<-_EOF
					I'll do what you say, but
					next time you should probably do 'drbd stop' first
					( or make this script more intelligent, so    )
					( this message is not triggered unnecessarily )

			_EOF
		fi
	fi

	# re-initialize, could be set by a failed attempt to use pre-parsed data
	CONF=(__hash__0)
	SEEN_RESOURCES=''

	debug 0 "sorry, this is bash, so parsing may take a while"
	local PARSE=""
	local TOKEN='' TOKEN_LINE='' TOKEN_LINE_NR=0
	if [ ! -f "$CONFIG" ]; then
	    echo "drbd not configured - $CONFIG is missing";
	    exit -1;
	fi

	while get_token; do
		case "$TOKEN" in
		global)
			parse_global_section
			continue ;;
		skip)
			get_token ; get_token ; skip_section
			continue ;;
		esac
		expect "resource"; get_token
		# resource name may be phony,
		# I restrict it to alphanums, dot, dash and underscore, though
		# QQQ should I allow umlauts?
		expect "+([._A-Za-z0-9-])"
		PARSE="resource '$TOKEN'"
		SEEN_RESOURCES="$SEEN_RESOURCES $TOKEN "
		parse_resource_section
		PARSE=""
	done < "$CONFIG" || { die "can not open <$CONFIG>"$'\n'; }
	echo
	CONFIG=''
	debug 2 "resources in $CONFIG:"
	debug 2 "  %s" $SEEN_RESOURCES
	[ $VERBOSITY -ge 2 ] && Dump CONF
	check_sanity
	if [ -z "$OPT_DRY_RUN" ] && [ "$COMMAND" != checkconfig ]
	then
		{   # store this config to avoid parsing again and again
			echo "SEEN_RESOURCES='$SEEN_RESOURCES'"
			echo "MINOR_COUNT='$MINOR_COUNT'"
			echo "MODULE_OPTIONS='$MODULE_OPTIONS'"
			Dump CONF
			echo "MD5SUM='$(md5sum < $CONFIG_PARSED)'"
		}  >| $CONFIG_PARSED \
		|| { die "cannot write to $CONFIG_PARSED"$'\n'; }
	else
		echo "parsed configuration NOT written to $CONFIG_PARSED"
	fi
}

read_parsed_config() # {{{2
# if we have a pre-parsed config, and the real config is not newer,
# and this script has not changed since, read it.
# if it seems self consistent, use it.
{
	trace0
	# has to exist ;)
	[ -e "$CONFIG_PARSED" ] \
|| return 1

	# remove these lines, once the preparsed stuff works.
	# make normal runs less noisy again
	# [[ -z $_SCRIPT_RECURSION ]] \
	# && echo "$link: trying pre-parsed config $CONFIG_PARSED"

	# pre-parsed should be newer than config, and newer than this script
	[ $CONFIG_PARSED -nt $CONFIG \
	-a $CONFIG_PARSED -nt $0 ] \
|| return 1

	source $CONFIG_PARSED
	# maybe someone accidentally changed *.parsed
	# if it was intentional, md5sum would have been updated.
	[ "$MD5SUM" == "$( grep -v ^MD5SUM= $CONFIG_PARSED | md5sum )" ] \
|| return 1

	CONFIG=''
	[ $VERBOSITY -ge 2 ] && Dump CONF
	# check_sanity
	debug 0 "preparsed config $CONFIG_PARSED seems consistent"
	return 0 # ok
}

check_one_parameter() # {{{2
{
	case $val in
		__implicit__*|__optional__*|__appended__*)
			return ;;

		__required__*)
			fetch CONF "$RES:$key" \
			|| { die "$PARSE: required '$key' not set"; }
			;;

		*)
			fetch CONF "$RES:$key" \
			|| store -new CONF "$RES:$key" ${val%% *}
			;;
	esac
}

check_if_all_parameters_set() # {{{2
{
	trace0 "for $RES"
	local i tmp host key val
	set -- "${TEMPLATE[@]}"
	shift # the hash magic
	let i=1 ; while [ $i -lt $# ]; do
		tmp=${!i} ; let i=i+1
		val=${!i} ; let i=i+1
		if mymatch "$tmp" "HOST_*" ; then
			for host in $HOSTS ; do
				key=${tmp/HOST_/on $host:}
				check_one_parameter
			done
		else
			key=$tmp
			check_one_parameter
		fi
	done
}

##################################################}}}1
#
# Sanity Checking
#
##################################################{{{1
#
# does some sanity checks over all resources
# - uniqueness of device and disk resources on this host
# - uniqueness of combination ip:port on all hosts
#
# XXX does NOT yet check
# - uniqueness of ip address over all hosts
#   (you will notice by other means for sure)
# - equivalence of config files on partner hosts
# - [ fill in what tests you think could make sense ]
#

#
# arbitrary digit math for bash :-)
#
multiply() # {{{2
# multiply <number> by <factor>
#   yupa, bash can do arbitrary digits math
#   well, unless factor is out of range too
# please use positiv integers, or you get unexpected results
{
	trace4
	RET_VAL=()
	[ -z "$1" -o -z "$2" ] && return
	# introduce new syntax, makes it readable
	mymatch "$1" "+([0-9])" && \
	[ "$2" == by ] && \
	mymatch "$3" "+([0-9])" \
	|| { die 'usage: multiply $N by $F'; }
	local N F R r
	N=$1 F=$3 r=0
	n=${N: -1}
	until [ -z "$N" -a -z "$r" ]; do
		N=${N%?}
		r=$(( ${n:-0} * $F + ${r:-0} ))
		R=${r: -1}$R
		r=${r%?}
		n=${N: -1}
	done
	RET_VAL=(${R##+(0)})
	debug 4 "$FUNCNAME returns (%s)" "${RET_VAL[*]}"
}

divide() # {{{2
# divide <nominator> by <denominator>
#   yupa, bash can do arbitrary digits math
#   well, unless denominator is out of range too
# please use positiv integers, or you get unexpected results
{
	trace4
	RET_VAL=()
	[ -z "$1" -o -z "$2" ] && return
	# introduce new syntax, makes it readable
	mymatch "$1" "+([0-9])" && \
	[ "$2" == by ] && \
	mymatch "$3" "+([0-9])" \
	|| { die 'usage: divide $nominator by $denominator'; }
	local N D F n f r
	N=$1 D=$3
	n=${N::1}
	while [ -n "$N" ]; do
		N=${N#?}
		f=$(( $n / $D ))       # current factor
		F=$F$f                 # cumulated factor
		r=$(( $n - $f * $D ))  # reminder
		n=${r#0}${N::1}
	done
	RET_VAL=(${F##+(0)})
	debug 4 "$FUNCNAME returns (%s)" "${RET_VAL[*]}"
}
#

check_disk() # {{{2
# underlying block device
{
	local size_in_kB conf_disk_size unit
	local size_in_sectors sector_size_in_byte
	trace "for $res on $host"
	local disk
	fetch CONF "$res:on $host:disk"; disk=$RET_VAL
	if fetch Disk "$disk"; then
		die "Disk $disk is used by $res($host) and $RET_VAL"$'\n'
	else
		store -new Disk "$disk" "$res($host)"
		# check the device size
		if ! size_in_sectors=$(blockdev --getsize $disk) ; then
			echo "$link: warning: could not get size of $disk"
return
		else if [ $size_in_sectors == 0 ] ; then
			echo "$link: warning $disk has zero size?"
return
		fi; fi
		if ! sector_size_in_byte=$(blockdev --getss $disk) ; then
			echo "$link: warning: could not get sector size of $disk, assuming 512"
			sector_size_in_byte=512
		fi
		multiply "$size_in_sectors" by "$sector_size_in_byte"
		# now byte -> kB
		divide "$RET_VAL" by 1024
		size_in_kB=$RET_VAL

		fetch CONF "$res:disk"
		set -- $RET_VAL
		while [ -n "$1" ] && ! mymatch "$1" "--disk-size=*" ; do shift ; done

		if [ -n "$1" ]; then
			conf_disk_size=$1
			# this is not to be returned, it is to make coding easier
			RET_VAL=(${1##*--disk-size=})
			RET_VAL=${RET_VAL%[kMG]}
			case "$conf_disk_size" in
			*G) multiply "$RET_VAL" by 1048576 ;; # 1024*1024
			*M) multiply "$RET_VAL" by 1024 ;;
			# default unit: kB
			esac
			conf_disk_size=$RET_VAL
			if [ ${#conf_disk_size} -gt ${#size_in_kB} ] \
			   || [ ${#conf_disk_size} -eq ${#size_in_kB} \
				  -a $conf_disk_size -gt $size_in_kB ]
			then
				die "configured $1 for '$res' is too large" \
					"\n    $conf_disk_size > $size_in_kB kB"
			else
				debug 0 "$res: $1: $conf_disk_size <= $size_in_kB"
			fi
		else
			echo "$link: disk-size not specified."
			echo "$link: you should set disk-size to no more than $size_in_kB"
		fi
	fi
}

check_device() # {{{2
# this is /dev/nb#
{
	trace "for $res on $host"
	local device
	fetch CONF "$res:on $host:device"; device=$RET_VAL
	if fetch Device "$device"; then
		die "Device $device is used by $res($host) and $RET_VAL"$'\n'
	else
		store -new Device "$device" "$res($host)"
	fi
}

timeout_exec() # {{{2
# executes the given arguments as command line, but kills
# that process if it does not return within the timeout
# defaults to 3 seconds, can be given as first option argument
# returns the exit code of the command,
# or 143 ("killed by SIGTERM") on timeout.
{
	local timeout=3
	mymatch "$1" "-+([0-9])" && { 
		timeout=${1#?}
		shift
	}
	bash -c '
			ex=143 kid=
			trap '\''test -n $kid && kill -TERM $kid; exit $ex'\'' TERM
			(sleep '"$timeout"'; kill -TERM $$) &> /dev/null &
			sleep_pid=$!
			"$@" &
			kid=$!
			wait $kid
			ex=$?
			kid=
			kill -TERM $sleep_pid
			exit $ex
			' -- "$@"
}

check_address_port() # {{{2
{
	trace "for $res on $host"
	local tmp
	fetch CONF "$res:on $host:address"; tmp=$RET_VAL
	$PING $tmp &> /dev/null
	[ $? -eq 1 -o $? -eq 143 ] && echo "$link: WARN: $host [$tmp] does not respond to ping"

	fetch CONF "$res:on $host:port"
	tmp="$tmp:$RET_VAL"
	if fetch Address_Port "$tmp" ; then
		die "Adress/port $tmp is used by $res($host) and $RET_VAL"$'\n'
	else
		store -new Address_Port "$tmp" "$res($host)"
	fi
}

check_sanity() # {{{2
{
	trace "for $SEEN_RESOURCES"
	local res host
	declare -a Disk=(__hash__0)
	declare -a Device=(__hash__0)
	declare -a Address_Port=(__hash__0)

	[[ $SEEN_RESOURCES ]] || { die "No resources configured."$'\n'; }
	for res in $SEEN_RESOURCES; do
		fetch CONF "$res:HOSTS"
		for host in $RET_VAL ; do
			# network resource check on all hosts
			check_address_port
			[ "$host" == "$HOST" ] || continue
			# disk and device check only on localhost
			check_disk
			check_device
		done
	done
	debug 0 "config seems sane: good ..."
}

##################################################}}}1
#
# command helpers
#
##################################################{{{1

needs_to_be_root() # {{{2
{
	[ -z "$OPT_DUMMY" ] && [ -z "$OPT_DRY_RUN" ] && [ -z "$OPT_CHECK_ONLY" ] || return
	if [ "$EUID" != 0 ] ; then
		echo "$link: you have to be root to $COMMAND"
		exit -1
	fi
}

# once I do error checking on each simple command, I can leave it on
# -e during execution, too
trap - EXIT
set +e

MINOR_COUNT=''
MODULE_OPTIONS=''

assure_module_is_loaded() # {{{2
# assure_module_is_loaded <device> <device> ...
# count of positional arguments is used as
# minor_count parameter to modprobe
{
	trace4
	[ -e "$PROC_DRBD" ] && return

	test -z "$MINOR_COUNT" && MINOR_COUNT=$#
	if [ "$MINOR_COUNT" -lt $# ]; then
		echo "set minor_count parameter to number of seen resources: $#"
		MINOR_COUNT=$#
	fi
	myexec $MODPROBE -s drbd minor_count=$MINOR_COUNT $MODULE_OPTIONS \
	|| { die "Can not load the drbd module."$'\n'; }
	# tell klogd to reload module symbol information ...
	[ -e /var/run/klogd.pid ] && [ -x /sbin/klogd ] && /sbin/klogd -i
}

mysuspend() # {{{2
# suspends the current process
# the "suspend" builtin only works with job control enabled,
# which I do not like for scripts
{ local mypid; read mypid _ </proc/self/stat; kill -STOP $mypid; }

mykill() # {{{2
# mykill -SIGSPEC pid1 [...]
# kills SIGSPEC to all pids, IFF they are my children.
{
	local sigspec mypid kid ppid
	if [ ${1::1} = "-" ] ; then
		sigspec=$1; shift
	else
		sigspec="-TERM"
	fi
	# I might be a subshell already
	read mypid _ </proc/self/stat
	for kid in "$@"; do
		test -r /proc/$kid/stat &&
		read _ _ _ ppid _ </proc/$kid/stat || continue
		[ "$ppid" = $mypid ] && kill $sigspec $kid
	done
}

myexec() # {{{2
# executes the arguments as commandline
# logs what is executed, and logs the output on nonzero exit code
# $errtxt is global, so the caller can match on it
{
	local ret
	debug 1 "%s" "$*"
	if [ -n "$OPT_DRY_RUN" ]; then
		"$@" ; return 0
	else
		errtxt=$("$@" 2>&1) && return
	fi
	ret=$?
	error "$@ [$ret]:"
	error "${errtxt:-NO OUTPUT}"
	return $ret
}

mount_status() # {{{2
# mount_status <device>
# RET_VAL=''|rw|ro
{
	trace4
	local device=$1
	mounts=$(</proc/mounts)
	IFS=$'\n' ; set -- $mounts ; IFS=$' \t\n'
	while [ -n "$1" ] && ! mymatch "$1" "$device *"; do shift; done

	set -- $1
	RET_VAL=(${4%%,*} $*)
	debug 4 "$FUNCNAME returns (%s)" "${RET_VAL[*]}"
	[ $# -gt 0 ]
}

get_drbd_status() # {{{2
# get_drbd_status <resource>
# parses /proc/drbd for line describing <resource>
# if <device> is not found, dies
#
# sets RET_VAL:
#	[0] connection status
#	[1] state of this host
#	[2] state of partner host
#
{
	trace
	RET_VAL=()
	fetch CONF "$1:on $HOST:device"
	local device=$RET_VAL
	local major minor line progress_line='' ls_pattern

	[ -b "$device" ] || { die "'$device' is not a block device"; }

	# (?) bash has no stat command.
	# use ls for determining device numbers
	line=$(/bin/ls -nl $device)
	ls_pattern="brw-[r-][w-]-[r-]--+( )+([0-9])+( )+([0-9])+( )+([0-9])+( )+([0-9]),+( )+([0-9])*$device"
	mymatch "$line" "$ls_pattern" \
	|| { die "ls '$device':\n$line\nparse error, could be wrong permissions" ; }

	# split $line on white space into $*
	set -- $line
	major=${5%,}
	minor=$6

	# QQQ check for device major, warn if unexpected. maybe should even die?
	#     can this happen? or does drbd refuse to configure such a device...
	[ "$major" == 43 ] || echo "$link: WARN: $device major is not 43; you risk deadlock"

	IFS=$'|'
	RET_VAL=($(sed -ne "/^$minor:/"'{
		s/^.*cs://;            # remove prefix
		s/ st:/|/;s/\//|/;     # separate with IFS
		s/ .*//;               # erease statistics
		h;                     # else hold
		${b print_and_quit;}
				# print if no continuation line follows (/^\t/!)
		n;/^'$'\t''/!{b print_and_quit;};
		H;                     # else append
		${b print_and_quit;};  # if last line, print
				# second continuation line?
		n;/^'$'\t''/!{b print_and_quit};
		H;b print_and_quit;    # append that, too, print and quit.
	}
	b;
	:print_and_quit
	x;s/'$'\\\n''/|/;p;q;
	' <$PROC_DRBD ))
	debug 4 "$FUNCNAME returns (%s)" "${RET_VAL[*]}"
	IFS=$' \t\n'
	[ -n "$RET_VAL" ] || error "Minor for $device '$res' not found in $PROC_DRBD"
	[ -n "$RET_VAL" ] # this is the return code
}

connection_status() # {{{2
# connection_status <resource>
{
	trace0
	RET_VAL=()
	get_drbd_status $1 || return -1
	if [ "${RET_VAL[1]}" == Primary \
	-a "${RET_VAL[2]}" == Primary ] ; then
		RET_VAL=("error")
		debug 4 "$FUNCNAME returns (%s)" "${RET_VAL[*]}"
return
	fi
	case $RET_VAL in
		Unconfigured  |\
		StandAlone    |\
		Unconnected   |\
		Timeout       |\
		BrokenPipe    |\
		WFConnection  |\
		WFReportParams)
			RET_VAL=("stopped") ;;

		Connected     |\
		SyncPaused    |\
		SyncingAll    |\
		SyncingQuick  )
			RET_VAL=("running" "${RET_VAL[3]}") ;;
	esac
	debug 4 "$FUNCNAME returns (%s)" "${RET_VAL[*]}"
}

datadisk_status() # {{{2
# datadisk_status <resource>
{
	trace4
	local res=$1 mystate device mounted tmp
	RET_VAL=()
	get_drbd_status $res  || return -1
	mystate=${RET_VAL[1]}
	fetch CONF "$res:on $HOST:device"; device=$RET_VAL
	mount_status $device; mounted=$RET_VAL
	case $mystate in
		Primary   )
			if [ "${RET_VAL[2]}" == Primary ] ; then
				RET_VAL=("error")
			else
				case "$mounted" in
				rw) RET_VAL=("running");;
				ro) note "warning: $res $device Primary, but mounted readonly"
					RET_VAL=("running") ;;
				# report "stopped" when not mounted
				'') note "warning: $res $device Primary, but not mounted"
					RET_VAL=("stopped") ;;
				*)  error "warning: unexpected RET_VAL of mount_status: '$mounted'"
					RET_VAL=("stopped") ;;
				esac
			fi ;;

		Unknown   |\
		Secondary )
			RET_VAL=("stopped") ;;
	esac
	debug 4 "$FUNCNAME returns (%s)" "${RET_VAL[*]}"
}

configure() # {{{2
# configure <device>
# implements drbd start
{
	trace0
	local res=$1
	local errtxt device command host

	get_drbd_status $res || return -1
	if [ -z "$OPT_DUMMY" -a -z "$OPT_DRY_RUN" ] \
		&& [ $RET_VAL != Unconfigured ]
	then
		echo
		note "WARN: '$res' already configured. I'll do what you say,"$'\n' \
			 "  but it should probably be '$link $res restart'"
		echo
	fi

	fetch CONF "$res:on $HOST:device"
	device=$RET_VAL

	command="$DRBD_SETUP $device disk"
	fetch CONF "$res:on $HOST:disk"
	command="$command $RET_VAL"

	fetch CONF "$res:disk"
	command="$command $RET_VAL"

	echo -n "Setting up '$res' .. disk "
	myexec $command && echo -n "ok .. " || return -1
	reconnect $res &&
	echo "OK"
}

reconnect() # {{{2
# reconnect <resource>
# implements drbd reconnect
# called by configure, too
{
	trace0
	local res=$1
	local errtxt host command

	get_drbd_status $res || return -1
	if [ -z "$OPT_DUMMY" -a -z "$OPT_DRY_RUN" ] \
		&& [ "$RET_VAL" == @(Connected|Sync*) ]
	then
		echo -e "\n$link: WARN: '$res' is in $RET_VAL state."
		echo -e "$link: I'll do what you say, but why do you reconnect?\n"
	fi

	# build up the command line; do it piece by piece {{{3
	command=$DRBD_SETUP
	host=$HOST

	fetch CONF "$res:on $host:device"
	command="$command $RET_VAL net"

	fetch CONF "$res:on $host:address"
	command="$command $RET_VAL"
	fetch CONF "$res:on $host:port"
	command="$command:$RET_VAL"

	fetch CONF "$res:HOSTS"
	set -- ${RET_VAL/ $host /}
	host=$1

	fetch CONF "$res:on $host:address"
	command="$command $RET_VAL"
	fetch CONF "$res:on $host:port"
	command="$command:$RET_VAL"

	fetch CONF "$res:protocol"
	command="$command $RET_VAL"

	fetch CONF "$res:net"
	command="$command $RET_VAL" # }}}3
	echo -n "net .. "
	myexec $command
}

become_active() # {{{2
# become_active <resource>
{
	trace0
	local res=$1 ret tmp
	local errtxt device command fsckcmd mounts incondegcmd

	fetch CONF "$res:on $HOST:device"
	device=$RET_VAL

	get_drbd_status $res || return -1
	if [ "${RET_VAL[0]}" == Unconfigured ] ; then
		echo "$link: '$res' is not configured"
		echo "$link: please call 'drbd $res start' first"
		# TODO could try to configure right here myself.
		# but that involves too much, since I should wait for connection and so on.
		# or I could call out to drbd start here. but why not educate the user?
		# The bug was in drbd(setup) anyways, since drbd sets its state to
		# Primary happily on an unconfigured device!
		# update: last statement is no longer true, that has been fixed in the module...
return -1
	fi
	if [ "${RET_VAL[1]}" == Primary ] ; then
		[ "${RET_VAL[2]}" == Primary ] \
		&& { error "Both hosts in Primary state on '$res'"; return -1; }
		echo "$link: '$res' already in Primary state, checking mount status"
	else
		# try to become primary, die on error
		command="$DRBD_SETUP $device primary"
		echo -n "Setting '$res' to Primary .. "
		myexec $command && echo "OK"
		ret=$?
		if [ $ret = 21 ]; then
			fetch CONF "$res:incon-degr-cmd"
			incondegrcmd=$RET_VAL
			[ "$incondegrcmd" = "0" ] ||
			myexec $incondegrcmd || { error "incondegrcmd FAILED: exit code $?"; return -1; }
		fi
	fi

	# ok, primary now
	# is device mounted?
	mount_status $device
	if [ -z "$RET_VAL" ] ; then
		fetch CONF "$res:fsckcmd"
		fsckcmd=$RET_VAL
		if [ -n "$OPT_DUMMY" -a -n "$OPT_DRY_RUN" ] ; then
			fsckcmd="echo # $fsckcmd"
		fi
		echo -n "$link: '$res' fsck $device .. "
		myexec $fsckcmd $device; ret=$?
			case $ret in
			0) echo "OK" ;;
			1) note "$fsckcmd found and corrected some errors";;
			*) error "$fsckcmd $device FAILED"$'\n';; 
			esac
		echo -ne "$link: '$res' mounting $device .. "
		myexec $MOUNT $device && echo "OK" || return $?
	else
		if [ "$RET_VAL" == rw ] ; then
			tmp=rw
		else
			tmp="ro, NOT remounted"
			# QQQ should we try to remount it?
			#
			# debug 1 "$MOUNT $device -o remount,rw"
			# $MOUNT $device -o remount,rw
		fi
		error "$device already mounted $tmp"
	fi
}

become_passive() # {{{2
# become_passive <resource>
# RET_VAL=($errtxt $device)
{
	trace0
	local res=$1
	local errtxt status device command

	get_drbd_status $res || return 10
	status=("${RET_VAL[@]}")
	fetch CONF "$res:on $HOST:device"
	device=$RET_VAL

	RET_VAL=("???" "$device")

	[ "$status" == Unconfigured ] && return 10
	[ "${status[1]}" == Secondary ] &&
		case $status in
		Sync*) return 20;;
		*)     return 11;;
		esac
	[ "${status[1]}" == Primary ] &&
		case $status in
		Sync*)
			if $dontwait; then
				note "Can not stop $res: $status in progress"
				return 1
			else
				note "$status in processes on $res. I'll wait for this to finish."
				trap 'read mypid _ < /proc/self/stat ; [ $mypid == $$ ] || exit' INT
				myexec $DRBD_SETUP $device wait_sync ; ret=$?
				case $ret in
				0)	:;; # ok, now cs:Connected again
				1)
					note "'$res' wait_sync timed out. you probably have link problems."
					return -1;;
				2) 
					note "'$res' wait_sync not connected. you probably have link problems."
					return -1;;
				*)
					note "'$res' wait_sync terminated unexpectedly"
					return -1;;
				esac
			fi ;;	
		esac

	mount_status $device
	if [ -n "$RET_VAL" ]; then
		note "'$res' $device is mounted on ${RET_VAL[2]}, trying to unmount"
		if ! $UMOUNT $device &> /dev/null ; then
			if ! $dontkill ; then
				# umount failed, kill all processes accessing <device>
				note "'$res' trying to kill users of $device"
				myexec $FUSER -k -m $device > /dev/null
			fi
			sleep 3 # hopefully the signals get delivered within this time
			# try again
			myexec $UMOUNT $device || {
				# failed again, FATAL
				error "$UMOUNT $device FAILED"
				return -1
			}
		fi
		echo "$link: '$res' $device unmounted"
	fi

	# ok, <device> now unmounted,
	# try to become secondary, return on success
	command="$DRBD_SETUP $device secondary"
	echo -ne "$link: '$res' becomming Secondary .. "
	myexec $command && echo "OK" && return

	# could not become secondary
	# we might be in resync
	# error will be handled by caller
	# errtxt is set by myexec
	if mymatch "$errtxt" "?(*$'\n')Resynchroni*" ; then
		RET_VAL=("$errtxt" $device)
		return 1
	fi

	# drbdsetup secondary failed
	# and we are not in resync.
	# FATAL
	error "'$command' FAILED"
	return -1
}


wait_until_partner_passive() # {{{2
# wait_until_partner_passive <resource>
{
	trace0
	local res=$1 device

	get_drbd_status $res || return -1
	if [ "${RET_VAL[2]}" == Primary ] ; then
	# QQQ should we send secondary_remote?
		# fetch CONF "$res:on $HOST:device"
		# device=$RET_VAL
		# debug 1 "$DRBD_SETUP $device secondary_remote"
		# $DRBD_SETUP $device secondary_remote
		echo "Waiting until partner node is secondary on '$res'; <CTRL-C> for abort"
		get_drbd_status $res
		until [ "${RET_VAL[2]}" != Primary ] ; do
			sleep 5
			# $DRBD_SETUP $device secondary_remote
			get_drbd_status $res
		done
	fi
}

wait_for_connect() # {{{2
# wait_for_connect <resource>
# if USR1 is caught, force primary
# waits for connection, optional timeout
{
	trace
	res=$1
	device= timeout= ret=
	primary_on_timeout=true
	force_primary=false

	fetch CONF "$res:on $HOST:device"
	device=$RET_VAL

	fetch CONF "$res:inittimeout"
	timeout=${RET_VAL#-}
	[ ${RET_VAL::1} = "-" ] && primary_on_timeout=false

	( $DRBD_SETUP $device wait_connect -t $timeout ) &
	debug 1 "$DRBD_SETUP $device wait_connect -t $timeout"

	trap "kill -USR1 $! ; force_primary=true" USR1 # propagate
	trap "kill -TERM $! ; exit 0" TERM
	wait $! ; ret=$?

	case $ret in
	0) # connection established
		debug 0 "$link: '$res' connection established"
		;;
	1) # timeout
		note "WARNING: '$res' connection timed out"
		if $primary_on_timeout ; then
			note "forcing primary status on $device --"\
				"DATA INTEGRITY MAY BE COMPROMISED."
			myexec $DRBD_SETUP $device primary --timeout-expired || return
		else
			note "I'll let the cluster manager decide on '$res'..."
		fi
		;;
	2) # < Unconnected, Standalone, whatever...
		note "WARNING: '$res' no connection, and not trying to reconnect!"
		note "I'll let the cluster manager decide on '$res'..."
		;;
	*) # killed by signal or unexpectedly
		if $force_primary ; then
			note "Operator forced primary status on $res ($device)"
			myexec $DRBD_SETUP $device primary --human || return
		else
			note "drbdsetup $device wait_connect: unexpected exit code $ret"
		fi
		;;
	esac
}

wait_for_sync() # {{{2
# wait_for_sync <resource>
# waits for synchronization to finish
# may be interrupted by CTRL-C
{
	trace
	local res=$1
	local device

	fetch CONF "$res:on $HOST:device"
	device=$RET_VAL

	get_drbd_status $res || return -1
	if ! mymatch "$RET_VAL" "Sync*" ; then
		debug 0 "\n$link: '$res' is not Syncing."
		return
	fi
	if [ "${RET_VAL[1]}" == Secondary ] ; then
		note "'$res' $RET_VAL, waiting for this to finish"

		myexec $DRBD_SETUP $device wait_sync ; ret=$?
		case $ret in
		0)	:;; # ok, now cs:Connected again
		1)
			note "'$res' wait_sync timed out. you probably have link problems."
			return -1;;
		2) 
			note "'$res' wait_sync not connected. you probably have link problems."
			return -1;;
		*)
			note "'$res' wait_sync terminated unexpectedly"
			return -1;;
		esac

		sleep 4
		note "'$res' $RET_VAL finished, issue $DRBD_SETUP $device secondary_remote"
		$DRBD_SETUP $device secondary_remote &> /dev/null
		# No error check here, on purpose!
	else
		note "'$res' $RET_VAL, but I have the good data. No need to wait."
		note "WARNING: a cluster manager should not failover yet!"
	fi
}

ask_for_abort() # {{{2
# expects to be run asynchronously
# no more wait_connects:       exit 0
# user hits CTRL-C:            exit 1
# if read failed (eof/no tty): exit 2
# YES (force primary):         exit 3 and kill -USR1 $$
{
	trace4
	# parent tells us to terminate.
	# this is the expected, "normal" way out of here 
	trap 'mykill $!; exit 0' TERM
	# CTRL-C ... just quit this script. no further action.
	trap 'mykill $!; exit 1' INT

	sleep 3 & wait
	prompt='Do you want to abort waiting for other server and make this one primary?'$'\n'
	while read -p "$prompt" ans || exit 2 ; do
		[ "$ans" == yes ] && break
		still_waiting=$(ps w | grep wait_connect | sed -ne 's/.*\(drbdsetup\)/\\t\1/p' )
		if [ -n "$still_waiting" ]; then
			echo -e "still waiting:\n$still_waiting"
		else
			exit 0
		fi
		prompt='Answer either "yes" or not at all: '
	done
	# "yes": force remaining to primary!
	kill -USR1 $$
	exit 3
}

connect_and_sync() # {{{2
#
# Wait for partner to get ready, but include timeout and operator dialog.
# The wait builtin accepts one particular pid to wait for, or none at
# all. In the latter case it waits for all children to finish before
# continuing.
# Start all wait_for_connect()s.
# Then, if we have wait_connects, start the user dialog.
# wait for every connect child in turn, explicitly.
# When all are connected, kill user dialog, and wait for sync to finish,
# unless configured otherwise.
#
# User dialog may kill the parent shell with USR1, if operator
# uses "force". This is trapped, and propagated to all still
# waiting (for connection) children, which then force their
# devices into Primary state.
#
{
	trace0
	local res userdlg eta_kid kid wait_kids

	# for bash 2.02, INT is captured, but the trap does not return,
	# sometimes not even execute!
	# for "()&" processes, tty generated INT is delivered anyways,
	# no need to propagate
	trap 'echo "got CTRL-C... bye..."; trap - INT; exit 0' INT
	trap 'mykill -USR1 $wait_kids' USR1 # <==- on "YES", force primary! 
	# make sure we leave no orphans
	trap 'mykill -TERM $userdlg $wait_kids $eta_kid' EXIT

	wait_kids="" 
	for res in $DO_RESOURCES ; do
		# wait_for_connect, unless load-only
		fetch CONF "$res:load-only"
		if mymatch "$RET_VAL" "@(yes|on|true|1)"; then
			debug 0 "$link: load-only for '$res'"
			continue
		fi
		( wait_for_connect $res ) &
		wait_kids="$wait_kids $!"
	done
	if [ -n "$wait_kids" ]; then
		( ask_for_abort ) < $TERMINAL &> $TERMINAL &
		userdlg=$!
		for kid in $wait_kids; do
			wait $kid &> /dev/null
		done
		mykill -TERM $userdlg # dialog may already be dead
		wait $userdlg; ret=$?
	else
return 0 # no wait connects, nothing to do.
	fi

	case $ret in
	0|143)	echo "$link: $DO_RESOURCES done.";;
	1|130)	: ;; # CTRL-C; trap exits
	2)	note "read error on $TERMINAL...";;

# for some reason answering YES to the user dialog causes ret to be -1 on
# bash-2.05b.0(1); all other common versions I tested (2.{02,04,05}), return
# the correct exit code ret=3. I was not able to reproduce this behaviour with
# anything simpler than this original script, so I cannot spot the bug.
# anyways, I add -1 here to quiten the "unexpected" message.

	3|-1)	: ;; # force primary, done by the trap on USR1
	*)	note "user dialog has unexpected exit code $ret"
	esac

	wait # in case someone still wants to be reaped

	# now that we are all connected, wait for sync
	wait_kids="" userdlg=""
	for res in $DO_RESOURCES ; do
		# wait for sync, unless load-only or skip-wait
		fetch CONF "$res:load-only"
		if mymatch "$RET_VAL" "@(yes|on|true|1)"; then
			continue
		fi
		fetch CONF "$res:skip-wait"
		if mymatch "$RET_VAL" "@(yes|on|true|1)"; then
			echo "$link: skip-wait for '$res'"
		else
			( wait_for_sync $res ) &
			wait_kids="$wait_kids $!"
		fi
	done
	(	# start a "clock tick" to show ETA
		# first after 10 seconds, then every 5 minutes
		# do not leave orphan sleep lying around
		trap 'mykill $!; exit 0;' INT TERM
		sleep 10 & wait
		while true; do
			date
			sed '1,2d' $PROC_DRBD
			echo "Waiting for Sync to finish ..."
			sleep 300 & wait
		done
	) &
	eta_kid=$!
	for kid in $wait_kids; do
		wait $kid &> /dev/null
	done
	mykill -TERM $eta_kid

	wait # in case someone still wants to be reaped

	debug 0 "helpers done, checking whether we are actually Connected"
	for res in $DO_RESOURCES ; do
		get_drbd_status $res
		if [ "$RET_VAL" != Connected ] ; then
			echo "$link: '$res' still not in Connected state, but $RET_VAL"
			[ -n "${RET_VAL[3]}" ] && echo "${RET_VAL[3]}"  
		fi
	done
}

##################################################}}}1
#
# frame work for commandline options part II
#
##################################################{{{1
STARTDIR=`pwd`
COMMANDLINE="$link $*"
PARAMETERS=`$GETOPT "$@"` || print_usage_and_exit
# eval to strip possible quoting by getopt
eval "set -- $PARAMETERS"

dontwait=false # for now only used in 'stop'
dontkill=false # for now only used in 'stop'
while true; do
	case $1 in
	--dontwait) dontwait=true ;;
	--dontkill) dontkill=true ;;
	--dry-run) # {{{2
		OPT_DRY_RUN='yes'; shift
		[ -z "$_SCRIPT_RECURSION" ] \
		&& cat <<-_EOF

			--dry-run
			  I will only echo the commands that would have been executed.
			  Sequence of commands might be not identical to real run.

		_EOF
		MODPROBE="echo # modprobe"
		RMMOD="echo # rmmod"
		UMOUNT="echo # umount -v"
		MOUNT="echo # mount -v"
		FUSER="echo # fuser"
		TOUCH="echo # touch"
		LOGECHO="echo ### "
		LOGECHO="echo ### "

		DRBD_SETUP="echo # drbdsetup"
		PROC_DRBD="/proc/drbd"
		# CONFIG is set elsewhere, otherwise the dry run would be biased

		PING="echo # ping -c 1 -w 3"
		;;

	--dummy) # {{{2
		OPT_DUMMY='yes'
		# debugging of the script without running drbdsetup
		[ "$VERBOSITY" == "-1" ] && VERBOSITY=3
		cat <<-_EOF

			WARNING
			  debugging enabled, using dummy programs and file locations
			  you should not use this mode unless you have the files I point to

		_EOF
		DRBD_SETUP="$STARTDIR/try/mysleep" # this is a very simple C program
								 # to check if signal delivery works as intended.
								 # loop {sleep(1); printf(i);} on wait_{sync,connect}
								 # and simply returns on other
		MODPROBE="true"
		RMMOD="true"
		UMOUNT="true"
		MOUNT="true"
		FUSER="true"
		TOUCH="touch"       # otherwise --parse would not work
		LOGECHO="echo ### "
		LOGECHO="echo ### "

		# point to text files for debugging
		PROC_DRBD="$STARTDIR/try/proc.drbd"

		CONFIG=${OPT_CONFIG:-"$STARTDIR/try/drbd.conf"}
		[ "${CONFIG::1}" == "/" ] || CONFIG=$STARTDIR/$CONFIG_PARSED
		CONFIG_PARSED="$CONFIG.parsed"

		if ! [ -f $PROC_DRBD ] && [ -f $CONFIG ] && [ -x $DRBD_SETUP ] ; then
			CONFIG='';
			die "can not find all necessary files for dummy mode\nDid you mean --dry-run?"$'\n';
		fi

		[ $VERBOSITY -gt 4 ] && Dump TEMPLATE
		shift
		;;

	--parse) # {{{2
		OPT_FORCE_PARSE='yes'; shift
		;;

	--config*) # {{{2
		# in case getopt was not found, and user used --config=file syntax
		OPT_CONFIG=$1; shift
		mymatch "$OPT_CONFIG" "--config=*" && set -- "${OPT_CONFIG##--config=}" "$@"

		# option ordering on command line is no longer important with dummy mode
		OPT_CONFIG=$1
		CONFIG=$OPT_CONFIG
		# not standard config, must not be stored in /var/lib/drbd/
		CONFIG_PARSED=$CONFIG.parsed
		shift
		;;

	--info) # {{{2
		printf "%-14s: %s\n" BASH_VERSINFO "${BASH_VERSINFO[*]}"
		for v in CVSID CONTACT TERMINAL MODPROBE RMMOD UMOUNT MOUNT TOUCH \
			DRBD_SETUP PROC_DRBD CONFIG CONFIG_PARSED LOGECHO LOGERR
		do
			printf "%-14s: %s\n" $v "${!v:-# is not set}"
		done
		print_usage_and_exit
		;;

	# template # {{{2
		# --*)
		# 	OPT_*='yes'; shift
		# 	# do something useful
		# 	;;

	--)	# end options # {{{2
		shift; break
		;;

	-*) # {{{2
		# if no getopt was available, do this here
		CONFIG='' # mute the in <CONFIG> line in die_at
		die "unknown option $1"$'\n'
		;;

	*)	break ;; # not an option, but real argument # }}}2
	esac
done

if $dontwait || $dontkill ; then
	[ "$COMMAND" == stop ] ||
	note "options --dontkill, --dontwait are only used with 'stop' command"
fi

# outside the loop, to make it independend of position
# if used together with --config or --dummy
[ -n "$OPT_FORCE_PARSE" ] && $TOUCH -c $CONFIG

##################################################}}}1

#
# "Main" Program, datadisk or drbd
# depending on basename of this script
#
######################################################

# prepare {{{1

debug 0 "CVS ${CVSID//\$/}\n\n$link $*\n"
[ $# == 2 ] && { RESOURCE=$1; shift; }
[ $# == 1 ] && { COMMAND=$1 ; shift; } \
|| print_usage_and_exit

# since I keep saying 'drbd start device'
# instead of 'drbd device start'
if mymatch "$RESOURCE" "@(start|stop|status|reconnect|checkconfig)" ; then
	debug 0 "swapping COMMAND and RESOURCE arguments"
	_COMMAND=$RESOURCE; RESOURCE=$COMMAND; COMMAND=$_COMMAND
	unset _COMMAND
fi
if mymatch "$COMMAND" "@(start|stop|restart|reconnect)" ; then
	needs_to_be_root
fi

#
# read pre-parsed config, or if this seems inconsistent,
# parse again
#
read_parsed_config || parse_config

# I do not want to kill myself in drbd stop
# but for restart I need to remember who I am
mymatch "$0" "/*" && DRBD=$0 || DRBD="$PWD/$0"
cd /

MAIN_LINE_NO=$LINENO # to get lineno right in die for very old bash

DO_RESOURCES=$SEEN_RESOURCES
if [ -n "$RESOURCE" ] ; then
	fetch CONF "$RESOURCE:HOSTS"
	[ -n "$RET_VAL" ] \
	|| { die "resource '$RESOURCE' not defined in config file"$'\n'"(defined:$SEEN_RESOURCES)"$'\n'; }
	DO_RESOURCES=" $RESOURCE "
fi

# for now, do not use alternate config files without dummy mode
if [ -n "$OPT_CONFIG" ] && [ -z "$OPT_DUMMY" ] ; then
	OPT_CHECK_ONLY='*'
	link=drbd
fi
# }}}1

case $link in

drbd)
	case $COMMAND in
	checkconfig|$OPT_CHECK_ONLY) # {{{1
		# on VERBOSITY level >= 2 this is done above already.
		[ $VERBOSITY -lt 2 ] && Dump CONF
		cat <<-_EOF

		  ${OPT_CHECK_ONLY:+Alternate config specified, but not in dummy mode,}
		  ${OPT_CHECK_ONLY:+forced behavior is drbd checkconfig.}
		  Configuration seems useable.
		  If it does not work, you might have found a bug in this script.

		_EOF
		;;

	restart)                     # {{{1
		export _SCRIPT_RECURSION=yes
		$DRBD \
			${OPT_CONFIG:+--config=$OPT_CONFIG} \
			${OPT_DUMMY:+--dummy} \
			${OPT_DRY_RUN:+--dry-run} \
			$RESOURCE stop \
		&& \
		$DRBD \
			${OPT_CONFIG:+--config=$OPT_CONFIG} \
			${OPT_DUMMY:+--dummy} \
			${OPT_DRY_RUN:+--dry-run} \
			$RESOURCE start
		;;

	force-reload)                     # {{{1
		export _SCRIPT_RECURSION=yes
		$DRBD \
			${OPT_CONFIG:+--config=$OPT_CONFIG} \
			${OPT_DUMMY:+--dummy} \
			${OPT_DRY_RUN:+--dry-run} \
			$RESOURCE stop \
		&& \
		$DRBD \
			${OPT_CONFIG:+--config=$OPT_CONFIG} \
			${OPT_DUMMY:+--dummy} \
			${OPT_DRY_RUN:+--dry-run} \
			$RESOURCE start
		;;

	start)                       # {{{1
		debug 1 "===> $COMMANDLINE <==="
		assure_module_is_loaded $SEEN_RESOURCES
		for RESOURCE in $DO_RESOURCES ; do
			configure $RESOURCE || DO_RESOURCES=${DO_RESOURCES// $RESOURCE /}
		done
		[ -d /var/lock/subsys ] \
		&& $TOUCH /var/lock/subsys/drbd
		connect_and_sync ;;

	stop)                        # {{{1
		[ -e $PROC_DRBD ] || { echo "$link stop: nothing to do."; exit 0; }
		debug 1 "===> $COMMANDLINE <==="
		FAILED=''
		for RESOURCE in $DO_RESOURCES ; do
			become_passive $RESOURCE
			case $? in
				 0) note "'$RESOURCE' de-activated";;
				10) echo "$link: '$RESOURCE' not configured";;
				11) echo "$link: '$RESOURCE' already Secondary";;
				 *) # Secondary sync target comes here, too
				FAILED="${FAILED:+$FAILED, }'$RESOURCE[${RET_VAL[1]}]'";;
			esac
		done
		if [ -n "$FAILED" ] ; then
			cat <<-_EOF
				$link: $FAILED: resynchronizing, NOT de-activated
				   If you /need/ to stop this, despite of likely data corruption, try one of
				   drbdsetup <device> disconnect # will end the syncer
				   drbdsetup <device> down       # will even unconfigure the device
				   You will have to reconnect/reconfigure of course.
				Module has NOT been unloaded.
				_EOF
			exit -1
		fi
		if [ "$DO_RESOURCES" == "$SEEN_RESOURCES" ] ; then
			myexec $RMMOD -s drbd || { die "Can not unload the drbd module. ($?)"$'\n'; }
			[ -f /var/lock/subsys/drbd ] && rm -f /var/lock/subsys/drbd
			note "module has been UNLOADED"
		else
			for RESOURCE in $DO_RESOURCES ; do
				fetch CONF "$RESOURCE:on $HOST:device"; device=$RET_VAL
				myexec $DRBD_SETUP $device down &&
				note "'$RESOURCE' teared down $device"
			done
			note "module has NOT been unloaded"
		fi ;;

	reconnect)                   # {{{1
		[ -e $PROC_DRBD ] || { die "$PROC_DRBD not found. Is drbd in kernel?"$'\n'; }
		debug 1 "===> $COMMANDLINE <==="
		for RESOURCE in $DO_RESOURCES ; do
			echo -n "Reconfiguring '$RESOURCE' .. "
			reconnect $RESOURCE && echo "OK"
		done ;;
		# QQQ should we? connect_and_sync ;;

	status)                      # {{{1
		[ -e $PROC_DRBD ] || { die "$PROC_DRBD not found. Is drbd in kernel?"$'\n'; }
		for RESOURCE in $DO_RESOURCES ; do
		# no short circuit, but list each resource in turn
			connection_status $RESOURCE
			echo "$RESOURCE: $RET_VAL"
			[ -n "${RET_VAL[1]}" ] && echo "${RET_VAL[1]}"
		done ;;

	*)	print_usage_and_exit ;;

	esac ;; # }}}1

datadisk)

	[ -e $PROC_DRBD ] || { die "$PROC_DRBD not found. Try drbd start first."$'\n'; }

	case $COMMAND in
	restart)                     # {{{1
		export _SCRIPT_RECURSION=yes
		$0 \
			${OPT_CONFIG:+--config=$OPT_CONFIG} \
			${OPT_DUMMY:+--dummy} \
			${OPT_DRY_RUN:+--dry-run} \
			$RESOURCE stop \
		&& \
		$0 \
			${OPT_CONFIG:+--config=$OPT_CONFIG} \
			${OPT_DUMMY:+--dummy} \
			${OPT_DRY_RUN:+--dry-run} \
			$RESOURCE start
		;;

	start)                       # {{{1
		debug 1 "===> $COMMANDLINE <==="
		for RESOURCE in $DO_RESOURCES ; do
			wait_until_partner_passive $RESOURCE || continue
			become_active $RESOURCE || continue
			note "'$RESOURCE' activated"
		done ;;

	stop)                        # {{{1
		debug 1 "===> $COMMANDLINE <==="
		FAILED=''
		for RESOURCE in $DO_RESOURCES ; do
			become_passive $RESOURCE
			case $? in
				 0) note "'$RESOURCE' de-activated";;
				10) echo "$link: '$RESOURCE' not configured";;
				11) echo "$link: '$RESOURCE' already Secondary";;
				20) echo "$link: '$RESOURCE' already Secondary, but sync target";;
				 *)
				FAILED="${FAILED:+$FAILED, }'$RESOURCE[${RET_VAL[1]}]'";;
			esac
		done
		if [ -n "$FAILED" ] ; then
			cat <<-_EOF
				$link: $FAILED: resynchronizing, NOT de-activated
				   If you /need/ to stop this, despite of likely data corruption, try one of
				   drbdsetup /dev/nb## disconnect # will end the syncer
				   drbdsetup /dev/nb## down       # will unconfigure the device
				   You will have to reconnect/reconfigure of course.
				_EOF
			exit -1
		fi ;;

	status)                      # {{{1
		for RESOURCE in $DO_RESOURCES ; do
		# no short circuit, but list each resource in turn
			datadisk_status $RESOURCE
			echo "$RESOURCE: $RET_VAL"
		done ;;

	*)	print_usage_and_exit
		;;
	esac ;; # }}}1

*)	print_usage_and_exit;;

esac

# Style, Comments, Details {{{1
######################################################

#l.g.e:
# My favorite editor is vim. I use tabstops of width 4 for indent.
# I dumped the indentation folding, and use markers (triple "{") now.
# (if you like vim, but don't know folds: help folding)
#
# Use global shell array RET_VAL for non $? (success/failure) return
# values.  Always use array assignment. If you refer to the
# basename you actually reference the [0] index, so this should be
# the main return value. Use additional array positions for
# additional info.
#
# The debug levels may not be consistent; level 1 is reserved for things that
# should go to the log file/syslog. Thus every important external program should
# have a "debug 1 $commandline"; myexec() is a wrapper for this.
#
# Use "local" variables in functions. These are visible to
# subsequently called helper functions.  It makes for less
# noisy code; but be careful, there are situations, where it is
# more clean to use function arguments.
#
# sometimes outdent returns in the middle of a function
#
# Try to use builtin commands or functions and minimal redirection, bash
# has lots of features you wanted to learn anyways ;)
#
# WARNING
# I hear people say some of the code involved in signal handling is not
# reentrant. if signals are badly timed, it may yield strange results
# update: I think this is is less an issue now... (April 2003)
#

#
# vim macros
#
# only works with :set hidden if file has been changed
# vim macro for ipell'ing comment blocks {{{2
# :e tmp.ispell
# dG/^[ 	]*#
# V/^[ 	]*\([^ 	#]\|$\)
# kdpkd1G:w!
# :!ispell %
# :e!
# 1GVGdPn

# vim macro for ipell'ing end commets {{{2
# :e tmp.ispell
# dG/^[ 	]*[^ 	][^#]\+ # /e
# d$P:w!
# :!ispell %
# :e!
# 0d$$pj0

# vim: set tw=0 nowrap ts=4 sw=4 ai foldmethod=marker foldcolumn=3 foldlevel=0 foldignore='' nofoldenable :
