Add fence_kdump support for generic clusters

Adds two new options to kdump.conf to be able to configure fence_kdump
support for generic clusters:

  fence_kdump_args <arg(s)>
    - Command line arguments for fence_kdump_send (it can contain all
      valid arguments except hosts to send notification to)

  fence_kdump_nodes <node(s)>
    - List of cluster node(s) separated by space to send fence_kdump
      notification to (this option is mandatory to enable fence_kdump)

Generic clusters fence_kdump configuration take precedence over older
method of fence_kdump configuration for Pacemaker clusters. It means
that if fence_kdump is configured using above options in kdump.conf, old
Pacemaker configuration is not used even if it exists.

Bug-Url: https://bugzilla.redhat.com/1078134
Signed-off-by: Martin Perina <mperina@redhat.com>
Acked-by: Vivek Goyal <vgoyal@redhat.com>

dracut-kdump.sh

@@ -246,6 +246,12 @@ read_kdump_conf()
         kdump_post)
             KDUMP_POST="$config_val"
             ;;
+        fence_kdump_args)
+            FENCE_KDUMP_ARGS="$config_val"
+            ;;
+        fence_kdump_nodes)
+            FENCE_KDUMP_NODES="$config_val"
+            ;;
         default)
             case $config_val in
                 shell)
@@ -289,20 +295,13 @@ read_kdump_conf()
 
 fence_kdump_notify()
 {
-    local nodes
+    if [ -n "$FENCE_KDUMP_NODES" ]; then
-
+        $FENCE_KDUMP_SEND $FENCE_KDUMP_ARGS $FENCE_KDUMP_NODES &
-    if [ -f $FENCE_KDUMP_NODES_FILE ]; then
-        if [ -f $FENCE_KDUMP_CONFIG_FILE ]; then
-            . $FENCE_KDUMP_CONFIG_FILE
-        fi
-
-        read nodes < $FENCE_KDUMP_NODES_FILE
-        $FENCE_KDUMP_SEND $FENCE_KDUMP_OPTS $nodes &
     fi
 }
 
-fence_kdump_notify
 read_kdump_conf
+fence_kdump_notify
 
 if [ -z "$CORE_COLLECTOR" ];then
     CORE_COLLECTOR=$DEFAULT_CORE_COLLECTOR

dracut-module-setup.sh

@@ -20,7 +20,7 @@ depends() {
         _dep="$_dep drm"
     fi
 
-    if is_pcs_fence_kdump; then
+    if [ is_generic_fence_kdump -o is_pcs_fence_kdump ]; then
         _dep="$_dep network"
     fi
 
@@ -285,7 +285,7 @@ kdump_install_conf() {
         esac
     done < /etc/kdump.conf
 
-    kdump_configure_fence_kdump
+    kdump_configure_fence_kdump  "/tmp/$$-kdump.conf"
     inst "/tmp/$$-kdump.conf" "/etc/kdump.conf"
     rm -f /tmp/$$-kdump.conf
 }
@@ -415,13 +415,9 @@ kdump_check_iscsi_targets () {
     }
 }
 
-
+# retrieves fence_kdump nodes from Pacemaker cluster configuration
-# setup fence_kdump in cluster
+get_pcs_fence_kdump_nodes() {
-# setup proper network and install needed files
-# also preserve '[node list]' for 2nd kernel /etc/fence_kdump_nodes
-kdump_configure_fence_kdump () {
     local nodes
-    is_pcs_fence_kdump || return 1
 
     # get cluster nodes from cluster cib, get interface and ip address
     nodelist=`pcs cluster cib | xmllint --xpath "/cib/status/node_state/@uname" -`
@@ -437,14 +433,51 @@ kdump_configure_fence_kdump () {
             continue
         fi
         nodes="$nodes $nodename"
-
-        kdump_install_net $nodename
     done
-    echo
 
-    echo "$nodes" > ${initdir}/$FENCE_KDUMP_NODES_FILE
+    echo $nodes
+}
+
+# retrieves fence_kdump args from config file
+get_pcs_fence_kdump_args() {
+    if [ -f $FENCE_KDUMP_CONFIG_FILE ]; then
+        . $FENCE_KDUMP_CONFIG_FILE
+        echo $FENCE_KDUMP_OPTS
+    fi
+}
+
+# setup fence_kdump in cluster
+# setup proper network and install needed files
+kdump_configure_fence_kdump () {
+    local kdump_cfg_file=$1
+    local nodes
+    local args
+
+    if is_generic_fence_kdump; then
+        nodes=$(get_option_value "fence_kdump_nodes")
+
+    elif is_pcs_fence_kdump; then
+        nodes=$(get_pcs_fence_kdump_nodes)
+
+        # set appropriate options in kdump.conf
+        echo "fence_kdump_nodes $nodes" >> ${kdump_cfg_file}
+
+        args=$(get_pcs_fence_kdump_args)
+        if [ -n "$args" ]; then
+            echo "fence_kdump_args $args" >> ${kdump_cfg_file}
+        fi
+
+    else
+        # fence_kdump not configured
+        return 1
+    fi
+
+    # setup network for each node
+    for node in ${nodes}; do
+        kdump_install_net $node
+    done
+
     dracut_install $FENCE_KDUMP_SEND
-    dracut_install -o $FENCE_KDUMP_CONFIG_FILE
 }
 
 # Install a random seed used to feed /dev/urandom

kdump-in-cluster-environment.txt

@@ -34,11 +34,11 @@ recovery service, fence_kdump_send will periodically send messages to all
 cluster nodes. When the fence_kdump agent receives a valid message from the 
 failed nodes, fencing is complete.
 
-How to configure cluster environment:
+How to configure Pacemaker cluster environment:
 
-If we want to use kdump in cluster environment, fence-agents-kdump should be 
+If we want to use kdump in Pacemaker cluster environment, fence-agents-kdump
-installed in every nodes in the cluster. You can achieve this via the following 
+should be installed in every nodes in the cluster. You can achieve this via
-command:
+the following command:
 
   # yum install -y fence-agents-kdump
 
@@ -61,6 +61,31 @@ Then enable stonith
 
 How to configure kdump:
 
-Actually there is nothing special in configuration between normal kdump and
+Actually there are two ways how to configure fence_kdump support:
-cluster environment kdump. So please refer to Kexec-Kdump-howto file for more
+
-information.
+1) Pacemaker based clusters
+     If you have successfully configured fence_kdump in Pacemaker, there is
+     no need to add some special configuration in kdump. So please refer to
+     Kexec-Kdump-howto file for more information.
+
+2) Generic clusters
+     For other types of clusters there are two configuration options in
+     kdump.conf which enables fence_kdump support:
+
+       fence_kdump_nodes <node(s)>
+            Contains list of cluster node(s) separated by space to send
+            fence_kdump notification to (this option is mandatory to enable
+            fence_kdump)
+
+       fence_kdump_args <arg(s)>
+            Command line arguments for fence_kdump_send (it can contain
+            all valid arguments except hosts to send notification to)
+
+     These options will most probably be configured by your cluster software,
+     so please refer to your cluster documentation how to enable fence_kdump
+     support.
+
+Please be aware that these two ways cannot be combined and 2) has precedence
+over 1). It means that if fence_kdump is configured using fence_kdump_nodes
+and fence_kdump_args options in kdump.conf, Pacemaker configuration is not
+used even if it exists.

kdump-lib.sh

@@ -5,7 +5,6 @@
 
 FENCE_KDUMP_CONFIG_FILE="/etc/sysconfig/fence_kdump"
 FENCE_KDUMP_SEND="/usr/libexec/fence_kdump_send"
-FENCE_KDUMP_NODES_FILE="/etc/fence_kdump_nodes"
 
 is_ssh_dump_target()
 {
@@ -38,6 +37,14 @@ is_pcs_fence_kdump()
     (pcs cluster cib | grep -q 'type="fence_kdump"') &> /dev/null || return 1
 }
 
+# Check if fence_kdump is configured using kdump options
+is_generic_fence_kdump()
+{
+    [ -x $FENCE_KDUMP_SEND ] || return 1
+
+    grep -q "^fence_kdump_nodes" /etc/kdump.conf
+}
+
 get_user_configured_dump_disk()
 {
     local _target

kdump.conf

@@ -123,6 +123,15 @@
 # dracut_args <arg(s)>
 #			- Pass extra dracut options when rebuilding kdump
 #			  initrd.
+#
+# fence_kdump_args <arg(s)>
+#			- Command line arguments for fence_kdump_send (it can contain
+#			all valid arguments except hosts to send notification to).
+#
+# fence_kdump_nodes <node(s)>
+# 			- List of cluster node(s) separated by space to send fence_kdump
+# 			notification to (this option is mandatory to enable fence_kdump).
+#
 
 #raw /dev/vg/lv_kdump
 #ext4 /dev/vg/lv_kdump
@@ -141,3 +150,5 @@ core_collector makedumpfile -l --message-level 1 -d 31
 #default shell
 #force_rebuild 1
 #dracut_args --omit-drivers "cfg80211 snd" --add-drivers "ext2 ext3"
+#fence_kdump_args -p 7410 -f auto -c 0 -i 10
+#fence_kdump_nodes node1 node2

kdump.conf.5

@@ -177,6 +177,21 @@ Kdump uses dracut to generate initramfs for second kernel. This option
 allows a user to pass arguments to dracut directly.
 .RE
 
+
+.B fence_kdump_args <arg(s)>
+.RS
+Command line arguments for fence_kdump_send (it can contain all valid
+arguments except hosts to send notification to).
+.RE
+
+
+.B fence_kdump_nodes <node(s)>
+.RS
+List of cluster node(s) separated by space to send fence_kdump notification
+to (this option is mandatory to enable fence_kdump).
+.RE
+
+
 .SH DEPRECATED OPTIONS
 
 .B net <nfs mount>|<user@server>

kdumpctl

@@ -166,7 +166,7 @@ function check_config()
 		case "$config_opt" in
 		\#* | "")
 			;;
-		raw|ext2|ext3|ext4|minix|btrfs|xfs|nfs|ssh|sshkey|path|core_collector|kdump_post|kdump_pre|extra_bins|extra_modules|default|force_rebuild|dracut_args)
+		raw|ext2|ext3|ext4|minix|btrfs|xfs|nfs|ssh|sshkey|path|core_collector|kdump_post|kdump_pre|extra_bins|extra_modules|default|force_rebuild|dracut_args|fence_kdump_args|fence_kdump_nodes)
 			[ -z "$config_val" ] && {
 				echo "Invalid kdump config value for option $config_opt."
 				return 1;
@@ -182,26 +182,38 @@ function check_config()
 			;;
 		esac
 	done < $KDUMP_CONFIG_FILE
+	
+	check_fence_kdump_config || return 1
+
 	return 0
 }
 
-# check_pcs_fence_kdump <image timestamp>
+# get_pcs_cluster_modified_files <image timestamp>
-# return 0 if fence_kdump is configured in Pacemaker cluster and kdump initrd needs to be rebuilt
+# return list of modified file for fence_kdump modified in Pacemaker cluster
-function check_pcs_fence_kdump()
+get_pcs_cluster_modified_files()
 {
 	local image_time=$1
-	local cib_time
+	local time_stamp
+	local modified_files
 
+	is_generic_fence_kdump && return 1
 	is_pcs_fence_kdump || return 1
 
-	cib_time=`pcs cluster cib | xmllint --xpath 'string(/cib/@cib-last-written)' - | \
+	time_stamp=`pcs cluster cib | xmllint --xpath 'string(/cib/@cib-last-written)' - | \
-		  xargs -0 date +%s --date`
+		xargs -0 date +%s --date`
 
-	if [ -z $cib_time -o $cib_time -le $image_time ]; then
+	if [ -n $time_stamp -a $time_stamp -gt $image_time ]; then
-		return 1
+		modified_files="cluster-cib"
 	fi
 
-	return 0
+	if [ -f $FENCE_KDUMP_CONFIG_FILE ]; then
+		time_stamp=`stat -c "%Y" $FENCE_KDUMP_CONFIG_FILE`
+		if [ "$time_stamp" -gt "$image_time" ]; then
+			modified_files="$modified_files $FENCE_KDUMP_CONFIG_FILE"
+		fi
+	fi
+
+	echo $modified_files
 }
 
 function check_rebuild()
@@ -240,7 +252,7 @@ function check_rebuild()
 	fi
 
 	#also rebuild when Pacemaker cluster conf is changed and fence kdump is enabled.
-	check_pcs_fence_kdump $image_time && modified_files="cluster-cib"
+	modified_files=$(get_pcs_cluster_modified_files $image_time)
 
 	EXTRA_BINS=`grep ^kdump_post $KDUMP_CONFIG_FILE | cut -d\  -f2`
 	CHECK_FILES=`grep ^kdump_pre $KDUMP_CONFIG_FILE | cut -d\  -f2`
@@ -249,10 +261,6 @@ function check_rebuild()
 	EXTRA_BINS="$EXTRA_BINS $CHECK_FILES"
 	files="$KDUMP_CONFIG_FILE $kdump_kernel $EXTRA_BINS"
 
-	if [ -f $FENCE_KDUMP_CONFIG_FILE ]; then
-		files="$files $FENCE_KDUMP_CONFIG_FILE"
-	fi
-
 	check_exist "$files" && check_executable "$EXTRA_BINS"
 	[ $? -ne 0 ] && return 1
 
@@ -573,6 +581,21 @@ function check_kdump_feasibility()
 	fi
 }
 
+function check_fence_kdump_config()
+{
+	local hostname=`hostname`
+	local nodes=$(get_option_value "fence_kdump_nodes")
+
+	for node in $nodes; do
+		if [ "$node" = "$hostname" ]; then
+			echo "Option fence_kdump_nodes cannot contain $hostname"
+			return 1
+		fi
+	done
+
+	return 0
+}
+
 function start()
 {
 	check_config