mdadm/test

#!/bin/bash
#
# run test suite for mdadm
dir=$(pwd)
mdadm=$dir/mdadm
testdir="tests"
targetdir="/var/tmp"
logdir="$targetdir"
config=/tmp/mdadm.conf

savelogs=0
exitonerror=1
prefix='[0-9][0-9]'

# use loop devices by default if doesn't specify --dev
DEVTYPE=loop
INTEGRITY=yes
LVM_VOLGROUP=mdtest

# make sure to test local mdmon, not system one
export MDADM_NO_SYSTEMCTL=1

# assume md0, md1, md2 exist in /dev
md0=/dev/md0
md1=/dev/md1
md2=/dev/md2
mdp0=/dev/md_d0
mdp1=/dev/md_d1

# We test mdadm on loop-back block devices.
# dir for storing files should be settable by command line maybe
size=20000
# super0, round down to multiple of 64 and substract 64
mdsize0=19904
# super00 is nested, subtract 128
mdsize00=19840
# super1.0 round down to multiple of 2, subtract 8
mdsize1=19992
mdsize1a=19988
mdsize12=19988
# super1.2 for linear: round to multiple of 2, subtract 4
mdsize1_l=19996
mdsize2_l=19996
# subtract another 4 for bitmaps
mdsize1b=19988
mdsize11=19992
mdsize11a=19456
mdsize12=19988

# ddf needs bigger devices as 32Meg is reserved!
ddfsize=65536

# $1 is optional parameter, it shows why to save log
save_log() {
	status=$1
	logfile="$status""$_basename".log

	cat $targetdir/stderr >> $targetdir/log
	cp $targetdir/log $logdir/$_basename.log
	echo "## $HOSTNAME: saving dmesg." >> $logdir/$logfile
	dmesg -c >> $logdir/$logfile
	$mdadm -As 2> /dev/null
	echo "## $HOSTNAME: saving proc mdstat." >> $logdir/$logfile
	cat /proc/mdstat >> $logdir/$logfile
	array=($(mdadm -Ds | cut -d' ' -f2))
	echo "## $HOSTNAME: mdadm -D ${array[@]}" >> $logdir/$logfile
	$mdadm -D ${array[@]} >> $logdir/$logfile
	[ "$1" == "fail" ] &&
		echo "FAILED - see $logdir/$_basename.log and $logdir/$logfile for details"
	# ignore saving external(external file, imsm...) bitmap
	cat /proc/mdstat | grep -q "linear\|external" && return 0
	if [ $DEVTYPE == 'lvm' ]
	then
		# not supported lvm type yet
		echo
	elif [ "$DEVTYPE" == 'loop' -o "$DEVTYPE" == 'disk' ]
	then
		if [ ! -z "$array" -a ${#array[@]} -ge 1 ]
		then
			md_disks=($($mdadm -D -Y ${array[@]} | grep "/dev/" | cut -d'=' -f2))
			cat /proc/mdstat | grep -q "bitmap"
			if [ $? -eq 0 ]
			then
				echo "## $HOSTNAME: mdadm -X ${md_disks[@]}" >> $logdir/$logfile
				$mdadm -X ${md_disks[@]} >> $logdir/$logfile
			fi
		else
			echo "## $HOSTNAME: no array assembled!" >> $logdir/$logfile
		fi
	fi
}

die() {
	echo -e "\n\tERROR: $* \n"
	save_log fail
	exit 2
}

cleanup() {
	udevadm settle
	$mdadm -Ssq 2> /dev/null
	case $DEVTYPE in
	loop )
		for d in 0 1 2 3 4 5 6 7  8 9 10 11 12 13
		do
			losetup -d /dev/loop$d
			rm -f /dev/disk/by-path/loop*
			rm -f /var/tmp/mdtest$d
		done
		;;
	lvm )
		for d in 0 1 2 3 4 5 6 7  8 9 10 11 12 13
		do
			eval "lvremove --quiet -f \$dev$d"
		done
		;;
	disk )
		$mdadm --zero ${disks[@]} &> /dev/null
		;;
	esac
}

ctrl_c() {
	exitonerror=1
}

do_setup() {
	trap cleanup 0 1 3 15
	trap ctrl_c 2

	[ -d $logdir ] || mkdir -p $logdir
	dmesg -c > /dev/null

	devlist=
	if [ "$DEVTYPE" == "loop" ]
	then
		# make sure there are no loop devices remaining.
		# udev started things can sometimes prevent them being stopped
		# immediately
		while grep loop /proc/partitions > /dev/null 2>&1
		do
			$mdadm -Ssq
			losetup -d /dev/loop[0-9]* 2> /dev/null
			sleep 0.2
		done
	elif [ "$DEVTYPE" == "disk" ]
	then
		if [ ! -z "$disks" ]
		then
			for d in $(seq 0 ${#disks[@]})
			do
				eval "dev$d=${disks[$d]}"
				eval devlist=\"\$devlist \$dev$d\"
				eval devlist$d=\"\$devlist\"
			done
			$mdadm --zero ${disks[@]} &> /dev/null
		else
			echo "Forget to provide physical devices for disk mode."
			exit 1
		fi
	fi
	for d in 0 1 2 3 4 5 6 7 8 9 10 11 12 13
	do
		sz=$size
		[ $d -gt 7 ] && sz=$ddfsize
		case $DEVTYPE in
		loop)
			[ -f $targetdir/mdtest$d ] ||
				dd if=/dev/zero of=$targetdir/mdtest$d count=$sz bs=1K > /dev/null 2>&1
			# make sure udev doesn't touch
			mdadm --zero $targetdir/mdtest$d 2> /dev/null
			[ -b /dev/loop$d ] || mknod /dev/loop$d b 7 $d
			if [ $d -eq 7 ]
			then
				losetup /dev/loop$d $targetdir/mdtest6 # for multipath use
			else
				losetup /dev/loop$d $targetdir/mdtest$d
			fi
			eval dev$d=/dev/loop$d
			eval file$d=$targetdir/mdtest$d
			;;
		lvm)
			unset MULTIPATH
			eval dev$d=/dev/mapper/${LVM_VOLGROUP}-mdtest$d
			if ! lvcreate --quiet -L ${sz}K -n mdtest$d $LVM_VOLGROUP
			then
				trap '' 0 # make sure lvremove is not called
				eval echo error creating \$dev$d
				exit 129
			fi
			;;
		ram)
			unset MULTIPATH
			eval dev$d=/dev/ram$d
			;;
		esac
		eval devlist=\"\$devlist \$dev$d\"
		eval devlist$d=\"\$devlist\"
		#" <-- add this quote to un-confuse vim syntax highlighting
	done
	path0=$dev6
	path1=$dev7
	ulimit -c unlimited
	[ -f /proc/mdstat ] || modprobe md_mod
	echo 2000 > /proc/sys/dev/raid/speed_limit_max
	echo 0 > /sys/module/md_mod/parameters/start_ro
}

# mdadm always adds --quiet, and we want to see any unexpected messages
mdadm() {
	rm -f $targetdir/stderr
	case $* in
	*-S* )
		udevadm settle
		p=`cat /proc/sys/dev/raid/speed_limit_max`
		echo 20000 > /proc/sys/dev/raid/speed_limit_max
		;;
	esac
	case $* in
	*-C* | *--create* | *-B* | *--build* )
		# clear superblock every time once creating or
		# building arrays, because it's always creating
		# and building array many times in a test case.
		for args in $*
		do
			[[ $args =~ "/dev/" ]] && {
				[[ $args =~ "md" ]] ||
					$mdadm --zero $args > /dev/null
			}
		done
		$mdadm 2> $targetdir/stderr --quiet "$@" --auto=yes
		;;
	* )
		$mdadm 2> $targetdir/stderr --quiet "$@"
		;;
	esac
	rv=$?
	case $* in
	*-S* )
		udevadm settle
		echo $p > /proc/sys/dev/raid/speed_limit_max
		;;
	esac
	cat >&2 $targetdir/stderr
	return $rv
}

# check various things
check() {
	case $1 in
	spares )
		spares=$(tr '] ' '\012\012' < /proc/mdstat | grep -c '(S)' || exit 0)
		[ $spares -ne $2 ] &&
			die "expected $2 spares, found $spares"
		;;
	raid* | linear )
		grep -sq "active $1 " /proc/mdstat ||
			die "active $1 not found"
		;;
	algorithm )
		grep -sq " algorithm $2 " /proc/mdstat ||
			die "algorithm $2 not found"
		;;
	resync | recovery | reshape )
		cnt=5
		while ! grep -sq $1 /proc/mdstat
		do
			if [ $cnt -gt 0 ] && grep -v idle /sys/block/md*/md/sync_action > /dev/null
			then # Something isn't idle - wait a bit
				sleep 0.5
				cnt=$[cnt-1]
			else
				die "no $1 happening"
			fi
		done
		;;
	nosync )
		sleep 0.5
		# Since 4.2 we delay the close of recovery until there has been a chance for
		# spares to be activated.  That means that a recovery that finds nothing
		# to do can still take a little longer than expected.
		# add an extra check: is sync_completed shows the end is reached, assume
		# there is no recovery.
		if grep -sq -E '(resync|recovery|reshape) *=' /proc/mdstat
		then
			incomplete=`grep / /sys/block/md*/md/sync_completed 2> /dev/null | sed '/^ *\([0-9]*\) \/ \1/d'`
			[ -n "$incomplete" ] &&
				die "resync or recovery is happening!"
		fi
		;;
	wait )
		p=`cat /proc/sys/dev/raid/speed_limit_max`
		echo 2000000 > /proc/sys/dev/raid/speed_limit_max
		sleep 0.1
		while grep -Eq '(resync|recovery|reshape|check|repair) *=' /proc/mdstat ||
			grep -v idle > /dev/null /sys/block/md*/md/sync_action
		do
			sleep 0.5
		done
		echo $p > /proc/sys/dev/raid/speed_limit_max
		;;
	state )
		grep -sq "blocks.*\[$2\]\$" /proc/mdstat ||
			die "state $2 not found!"
		sleep 0.5
		;;
	bitmap )
		grep -sq bitmap /proc/mdstat ||
			die "no bitmap"
		;;
	nobitmap )
		grep -sq "bitmap" /proc/mdstat &&
			die "bitmap present"
		;;
	readonly )
		grep -sq "read-only" /proc/mdstat ||
			die "array is not read-only!"
		;;
	inactive )
		grep -sq "inactive" /proc/mdstat ||
			die "array is not inactive!"
		;;
	# It only can be used when there is only one raid
	chunk )
		chunk_size=`awk -F',' '/chunk/{print $2}' /proc/mdstat | awk -F'[a-z]' '{print $1}'`
		if [ "$chunk_size" -ne "$2" ] ; then
			die "chunksize should be $2, but it's $chunk_size"
		fi
		;;
	* )
		die "unknown check $1"
		;;
	esac
}

no_errors() {
	if [ -s $targetdir/stderr ]
	then
		echo Bad errors from mdadm:
		cat $targetdir/stderr
		exit 2
	fi
}

# basic device test
testdev() {
	[ -b $1 ] || die "$1 isn't a block device."
	[ "$DEVTYPE" == "disk" ] && return 0
	udevadm settle
	dev=$1
	cnt=$2
	dvsize=$3
	chunk=$4
	if [ -z "$5" ]
	then
		mkfs.ext3 -F -j $dev > /dev/null 2>&1 && fsck -fn $dev >&2
	fi
	dsize=$[dvsize/chunk]
	dsize=$[dsize*chunk]
	rasize=$[dsize*2*cnt]
	# rasize is in sectors
	if [ -n "$DEV_ROUND_K" ]
	then
		rasize=$[rasize/DEV_ROUND_K/2]
		rasize=$[rasize*DEV_ROUND_K*2]
	fi
	[ `/sbin/blockdev --getsize $dev` -eq 0 ] && sleep 2
	_sz=`/sbin/blockdev --getsize $dev`
	[ $rasize -lt $_sz -o $[rasize*4/5] -gt $_sz ] &&
		die "size is wrong for $dev: $cnt * $dvsize (chunk=$chunk) = $rasize, not $_sz"
	return 0
}

rotest() {
	dev=$1
	fsck -fn $dev >&2
}

do_test() {
	_script=$1
	_basename=`basename $_script`
	if [ -f "$_script" ]
	then
		rm -f $targetdir/stderr
		# stop all arrays, just incase some script left an array active.
		$mdadm -Ssq 2> /dev/null
		mdadm --zero $devlist 2> /dev/null
		# this might have been reset: restore the default.
		echo 2000 > /proc/sys/dev/raid/speed_limit_max
		# source script in a subshell, so it has access to our
		# namespace, but cannot change it.
		echo -ne "$_script... "
		if ( set -ex ; . $_script ) &> $targetdir/log
		then
			dmesg | grep -iq "error\|call trace\|segfault" &&
				die "dmesg prints errors when testing $_basename!"
			echo "succeeded"
			_fail=0
		else
			save_log fail
			_fail=1
		fi
		[ "$savelogs" == "1" ] &&
			mv -f $targetdir/log $logdir/$_basename.log
		[ "$_fail" == "1" -a "$exitonerror" == "1" ] && exit 1
	fi
}

do_help() {
	cat <<-EOF
	Usage: $0 [options]
	Example for disk mode: ./test --dev=disk --disks=/dev/sda{2..15}
	Options:
		--tests=test1,test2,...     Comma separated list of tests to run
		--raidtype=                 raid0|linear|raid1|raid456|raid10|ddf|imsm
		--disable-multipath         Disable any tests involving multipath
		--disable-integrity         Disable slow tests of RAID[56] consistency
		--logdir=directory          Directory to save all logfiles in
		--save-logs                 Usually use with --logdir together
		--keep-going | --no-error   Don't stop on error, ie. run all tests
		--dev=loop|lvm|ram|disk     Use loop devices (default), LVM, RAM or disk
		--disks=                    Provide a bunch of physical devices for test
		--volgroup=name             LVM volume group for LVM test
		setup                       Setup test environment and exit
		cleanup                     Cleanup test environment
		prefix                      Run tests with <prefix>
		--help | -h                 Print this usage
	EOF
}

parse_args() {
	for i in $*
	do
		case $i in
		[0-9][0-9] )
			prefix=$i
			;;
		setup )
			echo "mdadm test environment setup"
			do_setup
			trap 0
			exit 0
			;;
		cleanup )
			cleanup
			exit 0
			;;
		--tests=* )
			TESTLIST=($(echo ${i##*=} | sed -e 's/,/ /g'))
			;;
		--raidtype=* )
			case ${i##*=} in
			raid0 )
				TESTLIST=($(ls $testdir | grep "[0-9][0-9]r0\|raid0"))
				;;
			linear )
				TESTLIST=($(ls $testdir | grep "linear"))
				;;
			raid1 )
				TESTLIST=($(ls $testdir | grep "[0-9][0-9]r1\|raid1" | grep -vi raid10))
				;;
			raid456 )
				TESTLIST=($(ls $testdir | grep "[0-9][0-9]r[4-6]\|raid[4-6]"))
				;;
			raid10 )
				TESTLIST=($(ls $testdir | grep "[0-9][0-9]r10\|raid10"))
				;;
			ddf )
				TESTLIST=($(ls $testdir | grep "[0-9][0-9]ddf"))
				;;
			imsm )
				TESTLIST=($(ls $testdir | grep "[0-9][0-9]imsm"))
				;;
			* )
				echo "Unknown argument: $i"
				do_help
				exit 1
				;;
			esac
			;;
		--logdir=* )
			logdir="${i##*=}"
			;;
		--save-logs )
			savelogs=1
			;;
		--keep-going | --no-error )
			exitonerror=0
			;;
		--disable-multipath )
			unset MULTIPATH
			;;
		--disable-integrity )
			unset INTEGRITY
			;;
		--dev=* )
			case ${i##*=} in
			loop )
				DEVTYPE=loop
				;;
			lvm )
				DEVTYPE=lvm
				;;
			ram )
				DEVTYPE=ram
				;;
			disk )
				DEVTYPE=disk
				;;
			* )
				echo "Unknown argument: $i"
				do_help
				exit 1
				;;
			esac
			;;
		--disks=* )
			disks=(${disks[*]} ${i##*=})
			;;
		--volgroup=* )
			LVM_VOLGROUP=`expr "x$i" : 'x[^=]*=\(.*\)'`
			;;
		--help | -h )
			do_help
			exit 0
			;;
		* )
			echo " $0: Unknown argument: $i"
			do_help
			exit 1
			;;
		esac
	done
}

check_env() {
	user=$(id -un)
	[ "X$user" != "Xroot" ] && {
		echo "test: testing can only be done as 'root'."
		exit 1
	}
	[ -x "raid6check" -a -x $mdadm ] || {
		echo "test: please run 'make everything' before perform testing."
		exit 1
	}
	cmds=(mdadm lsblk df udevadm losetup mkfs.ext3 fsck seq)
	for cmd in ${cmds[@]}
	do
		which $cmd > /dev/null || {
			echo "$cmd command not found!"
			exit 1
		}
	done
	mdadm_src_ver="$($mdadm -V 2>&1)"
	mdadm_sbin_ver="$($(which mdadm) -V 2>&1)"
	if [ "$mdadm_src_ver" != "$mdadm_sbin_ver" ]
	then
		# it's nessesary to 'make install' mdadm to /SBIN/DIR,
		# such as systemd/mdadm-grow-continue@.service, would
		# run as an instance by systemd when reshape happens,
		# thus ensure that the correct mdadm is in testing.
		echo "test: please run 'make install' before testing."
		exit 1
	fi
	if ! $(df -T . | grep -iq ext)
	then
		# 'external file' bitmap only supports with ext[2-4] file system
		echo "test: please run test suite with ext[2-4] file system."
		exit 1
	fi
	if $(lsblk -a | grep -iq raid)
	then
		# donot run mdadm -Ss directly if there are RAIDs working.
		echo "test: please run test suite without running RAIDs environment."
		exit 1
	fi
	# Check whether to run multipath tests
	modprobe multipath 2> /dev/null
	grep -sq 'Personalities : .*multipath' /proc/mdstat &&
		MULTIPATH="yes"
}

main() {
	check_env
	do_setup

	echo "Testing on linux-$(uname -r) kernel"
	[ "$savelogs" == "1" ] &&
		echo "Saving logs to $logdir"
	if [ "x$TESTLIST" != "x" ]
	then
		for script in ${TESTLIST[@]}
		do
			do_test $testdir/$script
		done
	else
		for script in $testdir/$prefix $testdir/$prefix*[^~]
		do
			do_test $script
		done
	fi

	exit 0
}

parse_args $@
main