configs: Update scripts and spec file with layout changes

With the configs in the new place, update the scripts to find
them and utilize the base-generic and generic heirarchy to apply
configs and overrides.

Implement new process_configs.sh script that post-process the
config changes in a simple script and remove those commands
from the spec file.  Add option flags to preserve functionality.

config_generation is a simple rename of baseconfig -> generic and
debugconfig -> debug and arm64 -> aarach64.

build_configs.sh is modified to find configs in generic and base-generic and
then apply base-generic first and if any generic files, apply those next.  The
generic directory is used as an overrides and is expected to be empty for
Fedora initially.

kernel.spec is modified to use process_configs.sh instead of all the
commands in the spec file.  Enabled spec options are translated to
script options.  The config manipulation is moved to be grouped
with all config manipulation commands.  This makes 'cd configs/' simpler.
Now all config scripts and executiion are done in configs/ directory.

v1 -> v2:
* the scripts were not working with SUBARCH correctly
* checkoptions was using wrong comparison file
* passing wrong kernel version to process_configs in spec file

v2 -> v3:
(incorporate Laura A's feedback)
* update README.txt
* fix build_configs.sh warnings
* Output info message on listnewconfig failure

README.txt

@@ -33,9 +33,9 @@ config heirarchy.
 Instead of having to maintain a config file for every arch variant we build on,
 the kernel spec uses a nested system of configs. Each option CONFIG_FOO is
 represented by a single file named CONFIG_FOO which contains the state (=y, =m,
-=n). These options are collected in the folder baseconfig. Architecture specifi
+=n). These options are collected in the folder base-generic. Architecture
-options are set in nested folders. An option set in a nested folder will
+specific options are set in nested folders. An option set in a nested folder
-override the same option set in one of the higher levels.
+will override the same option set in one of the higher levels.
 
 The individual CONFIG_FOO files only exist in the pkg-git repository. The RPM
 contains kernel-foo.config files which are the result of combining all the
@@ -45,10 +45,10 @@ script _must_ be run each time one of the options is changed.
 Example flow:
 
 # Enable the option CONFIG_ABC123 as a module for all arches
-echo "CONFIG_ABC123=m" > baseconfig/CONFIG_ABC1234
+echo "CONFIG_ABC123=m" > configs/base-generic/CONFIG_ABC1234
 # enable the option CONFIG_XYZ321 for only x86
-echo "# CONFIG_XYZ321 is not set" > baseconfig/CONFIG_XYZ321
+echo "# CONFIG_XYZ321 is not set" > configs/base-generic/CONFIG_XYZ321
-echo "CONFIG_XYZ321=m" > baseconfig/x86/CONFIG_XYZ321
+echo "CONFIG_XYZ321=m" > configs/base-generic/x86/CONFIG_XYZ321
 # regenerate the combined config files
 ./build_configs.sh
 
@@ -72,7 +72,7 @@ the status quo to:
 This is done to increase coverage testing, as not many people actually
 run kernel-debug.
 
-The debug options are managed in a separate heierarchy under debugconfig. This
+The debug options are managed in a separate heierarchy under base-debug. This
-works in a similar manner to baseconfig. More deeply nested folders, again,
+works in a similar manner to base-generic. More deeply nested folders, again,
 override options. The file config_generation gives a listing of what folders
 go into each config file generated.

configs/build_configs.sh

@@ -4,13 +4,33 @@
 # and debug to form the necessary $PACKAGE_NAME<version>-<arch>-<variant>.config
 # files for building RHEL kernels, based on the contents of a control file
 
-PACKAGE_NAME=kernel # defines the package name used
+PACKAGE_NAME="${1:-kernel}" # defines the package name used
+KVERREL="${2:-}"
+SUBARCH="${3:-}" # defines a specific arch for use with rh-configs-arch-prep target
+SCRIPT="$(readlink -f $0)"
+OUTPUT_DIR="$PWD"
+SCRIPT_DIR="$(dirname $SCRIPT)"
+
+# to handle this script being a symlink
+cd $SCRIPT_DIR
 
 set errexit
 set nounset
 
 control_file="config_generation"
 
+cleanup()
+{
+	rm -f config-*
+}
+
+die()
+{
+	echo "$1"
+	cleanup
+	exit 1
+}
+
 function combine_config_layer()
 {
 	dir=$1
@@ -29,14 +49,27 @@ function merge_configs()
 	archvar=$1
 	arch=$(echo "$archvar" | cut -f1 -d"-")
 	configs=$2
-	name=$PACKAGE_NAME-$archvar.config
+	name=$OUTPUT_DIR/$PACKAGE_NAME-$archvar.config
 	echo -n "Building $name ... "
 	touch config-merging config-merged
+
+	# apply base first
 	for config in $(echo $configs | sed -e 's/:/ /g')
 	do
+		perl merge.pl config-base-$config config-merging > config-merged
+		if [ ! $? -eq 0 ]; then
+			die "Failed to merge base"
+		fi
+		mv config-merged config-merging
+	done
+	for config in $(echo $configs | sed -e 's/:/ /g')
+	do
+		# not all override files exist
+		test -e config-$config || continue
+
 		perl merge.pl config-$config config-merging > config-merged
 		if [ ! $? -eq 0 ]; then
-			exit
+			die "Failed to merge configs"
 		fi
 		mv config-merged config-merging
 	done
@@ -60,10 +93,12 @@ function merge_configs()
 	echo "done"
 }
 
-glist=$(find baseconfig -type d)
+glist=$(find base-generic -type d)
-dlist=$(find debugconfig -type d)
+dlist=$(find base-debug -type d)
+gllist=$(test -d generic && find generic -type d)
+dllist=$(test -d debug && find debug -type d)
 
-for d in $glist $dlist
+for d in $glist $dlist $gllist $dllist
 do
 	combine_config_layer $d
 done
@@ -86,4 +121,15 @@ do
 	fi
 done < $control_file
 
-rm -f config-*
+# A passed in kernel version implies copy to final location
+# otherwise defer to another script
+if test -n "$KVERREL"
+then
+	for i in kernel-*.config
+	do
+		NEW="$(echo $i | sed "s/$PACKAGE_NAME-$SUBARCH/$PACKAGE_NAME-$KVERREL-$SUBARCH/")"
+		mv $i $NEW
+	done
+fi
+
+cleanup

configs/config_generation

@@ -4,33 +4,33 @@
 # config options, lowest priority to highest
 
 # x86_64
-x86_64=baseconfig:baseconfig-x86:baseconfig-x86-x86_64
+x86_64=generic:generic-x86:generic-x86-x86_64
-x86_64-debug=baseconfig:baseconfig-x86:baseconfig-x86-x86_64:debugconfig:debugconfig-x86:debugconfig-x86-x86_64
+x86_64-debug=generic:generic-x86:generic-x86-x86_64:debug:debug-x86:debug-x86-x86_64
 
 # i686
-i686=baseconfig:baseconfig-x86:baseconfig-x86-i686
+i686=generic:generic-x86:generic-x86-i686
-i686-debug=baseconfig:baseconfig-x86:baseconfig-x86-i686:debugconfig:debugconfig-x86
+i686-debug=generic:generic-x86:generic-x86-i686:debug:debug-x86
-i686-PAE=baseconfig:baseconfig-x86:baseconfig-x86-i686PAE
+i686-PAE=generic:generic-x86:generic-x86-i686PAE
-i686-PAEdebug=baseconfig:baseconfig-x86:baseconfig-x86-i686PAE:debugconfig:debugconfig-x86
+i686-PAEdebug=generic:generic-x86:generic-x86-i686PAE:debug:debug-x86
 
 # ppc64
-ppc64=baseconfig:baseconfig-powerpc:baseconfig-powerpc-powerpc64
+ppc64=generic:generic-powerpc:generic-powerpc-powerpc64
-ppc64-debug=baseconfig:baseconfig-powerpc:baseconfig-powerpc-powerpc64:debugconfig
+ppc64-debug=generic:generic-powerpc:generic-powerpc-powerpc64:debug
 
 # ppc64le
-ppc64le=baseconfig:baseconfig-powerpc:baseconfig-powerpc-powerpc64le
+ppc64le=generic:generic-powerpc:generic-powerpc-powerpc64le
-ppc64le-debug=baseconfig:baseconfig-powerpc:baseconfig-powerpc-powerpc64le:debugconfig
+ppc64le-debug=generic:generic-powerpc:generic-powerpc-powerpc64le:debug
 
 # s390x
-s390x=baseconfig:baseconfig-s390x
+s390x=generic:generic-s390x
-s390x-debug=baseconfig:baseconfig-s390x:debugconfig
+s390x-debug=generic:generic-s390x:debug
 
 # aarch64
-aarch64=baseconfig:baseconfig-arm:baseconfig-arm-arm64
+aarch64=generic:generic-arm:generic-arm-aarch64
-aarch64-debug=baseconfig:baseconfig-arm:baseconfig-arm-arm64:debugconfig:debugconfig-arm
+aarch64-debug=generic:generic-arm:generic-arm-aarch64:debug:debug-arm
 
 # arm
-armv7hl=baseconfig:baseconfig-arm:baseconfig-arm-armv7:baseconfig-arm-armv7-armv7
+armv7hl=generic:generic-arm:generic-arm-armv7:generic-arm-armv7-armv7
-armv7hl-debug=baseconfig:baseconfig-arm:baseconfig-arm-armv7:baseconfig-arm-armv7-armv7:debugconfig:debugconfig-arm
+armv7hl-debug=generic:generic-arm:generic-arm-armv7:generic-arm-armv7-armv7:debug:debug-arm
-armv7hl-lpae=baseconfig:baseconfig-arm:baseconfig-arm-armv7:baseconfig-arm-armv7-lpae
+armv7hl-lpae=generic:generic-arm:generic-arm-armv7:generic-arm-armv7-lpae
-armv7hl-lpae-debug=baseconfig:baseconfig-arm:baseconfig-arm-armv7:baseconfig-arm-armv7-lpae:debugconfig:debugconfig-arm
+armv7hl-lpae-debug=generic:generic-arm:generic-arm-armv7:generic-arm-armv7-lpae:debug:debug-arm

configs/process_configs.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+#
+# This script takes the merged config files and processes them through oldconfig
+# and listnewconfig
+
+
+die()
+{
+	echo "$1"
+	exit 1
+}
+
+# stupid function to find top of tree to do kernel make configs
+switch_to_toplevel()
+{
+	path="$(pwd)"
+	while test -n "$path"
+	do
+		test -d $path/firmware && \
+			test -e $path/MAINTAINERS && \
+			test -d $path/drivers && \
+			break
+
+		path="$(dirname $path)"
+	done
+
+	test -n "$path"  || die "Can't find toplevel"
+	echo "$path"
+}
+
+checkoptions()
+{
+	/usr/bin/awk '
+
+		/is not set/ {
+			split ($0, a, "#");
+			split(a[2], b);
+			if (NR==FNR) {
+				configs[b[1]]="is not set";
+			} else {
+				if (configs[b[1]] != "" && configs[b[1]] != "is not set")
+					 print "Found # "b[1] " is not set, after generation, had " b[1] " " configs[b[1]] " in Source tree";
+			}
+		}
+
+		/=/     {
+			split ($0, a, "=");
+			if (NR==FNR) {
+				configs[a[1]]=a[2];
+			} else {
+				if (configs[a[1]] != "" && configs[a[1]] != a[2])
+					 print "Found "a[1]"="configs[a[1]]" after generation, had " a[1]"="a[2]" in Source tree";
+			}
+		}
+	' $1 $2 > .mismatches
+
+	if test -s .mismatches
+	then
+		echo "Error: Mismatches found in configuration files"
+		cat .mismatches
+		exit 1
+	fi
+}
+
+function process_configs()
+{
+	# assume we are in $source_tree/configs, need to get to top level
+	pushd $(switch_to_toplevel)
+
+	for cfg in $SCRIPT_DIR/${PACKAGE_NAME}${KVERREL}${SUBARCH}*.config
+	do
+		arch=$(head -1 $cfg | cut -b 3-)
+		cfgtmp="${cfg}.tmp"
+		cfgorig="${cfg}.orig"
+		cat $cfg > $cfgorig
+
+		echo -n "Processing $cfg ... "
+
+		# an empty grep is good but leaves a return value, so use # 'true' to bypass
+		make ARCH=$arch KCONFIG_CONFIG=$cfg listnewconfig | grep -E 'CONFIG_' > .newoptions || true
+		if test -n "$NEWOPTIONS" && test -s .newoptions
+		then
+			echo "Found unset config items, please set them to an appropriate value"
+			cat .newoptions
+			rm .newoptions
+			exit 1
+		fi
+		rm .newoptions
+
+		make ARCH=$arch KCONFIG_CONFIG=$cfg oldnoconfig > /dev/null || exit 1
+		echo "# $arch" > ${cfgtmp}
+		cat "${cfg}" >> ${cfgtmp}
+		if test -n "$CHECKOPTIONS"
+		then
+			checkoptions $cfgtmp $cfgorig
+		fi
+		mv ${cfgtmp} ${cfg}
+		rm ${cfgorig}
+		echo "done"
+	done
+	rm "$SCRIPT_DIR"/*.config.old
+	popd > /dev/null
+
+	echo "Processed config files are in $SCRIPT_DIR"
+}
+
+NEWOPTIONS=""
+CHECKOPTIONS=""
+
+while [[ $# -gt 0 ]]
+do
+	key="$1"
+	case $key in
+		-n)
+			NEWOPTIONS="x"
+			;;
+		-c)
+			CHECKOPTIONS="x"
+			;;
+		*)
+			break;;
+	esac
+	shift
+done
+
+PACKAGE_NAME="${1:-kernel}" # defines the package name used
+KVERREL="$(test -n "$2" && echo "-$2" || echo "")"
+SUBARCH="$(test -n "$3" && echo "-$3" || echo "")"
+SCRIPT="$(readlink -f $0)"
+OUTPUT_DIR="$PWD"
+SCRIPT_DIR="$(dirname $SCRIPT)"
+
+# to handle this script being a symlink
+cd $SCRIPT_DIR
+
+process_configs

kernel.spec

@@ -478,7 +478,7 @@ Source39: kernel-x86_64-debug.config
 Source40: generate_all_configs.sh
 Source41: generate_debug_configs.sh
 
-Source42: check_configs.awk
+Source42: process_configs.sh
 
 # This file is intentionally left empty in the stock kernel. Its a nicety
 # added for those wanting to do custom rebuilds with altered config opts.
@@ -1225,6 +1225,24 @@ xzcat %{SOURCE5000} | patch -p1 -F1 -s
 git commit -a -m "Stable update"
 %endif
 
+# Note: Even in the "nopatches" path some patches (build tweaks and compile
+# fixes) will always get applied; see patch defition above for details
+
+git am %{patches}
+
+# END OF PATCH APPLICATIONS
+
+# Any further pre-build tree manipulations happen here.
+
+chmod +x scripts/checkpatch.pl
+
+# This Prevents scripts/setlocalversion from mucking with our version numbers.
+touch .scmversion
+
+# Deal with configs stuff
+mkdir configs
+cd configs
+
 # Drop some necessary files from the source dir into the buildroot
 cp $RPM_SOURCE_DIR/kernel-*.config .
 cp %{SOURCE1000} .
@@ -1255,25 +1273,9 @@ do
 done
 %endif
 
-# Note: Even in the "nopatches" path some patches (build tweaks and compile
-# fixes) will always get applied; see patch defition above for details
-
-git am %{patches}
-
-# END OF PATCH APPLICATIONS
-
-# Any further pre-build tree manipulations happen here.
-
-chmod +x scripts/checkpatch.pl
-
-# This Prevents scripts/setlocalversion from mucking with our version numbers.
-touch .scmversion
-
 # only deal with configs if we are going to build for the arch
 %ifnarch %nobuildarches
 
-mkdir configs
-
 %if !%{debugbuildsenabled}
 rm -f kernel-%{version}-*debug.config
 %endif
@@ -1291,31 +1293,21 @@ CheckConfigs() {
 }
 
 cp %{SOURCE42} .
-# now run oldconfig over all the config files
+OPTS=""
-for i in *.config
-do
-  cat $i > temp-$i
-  mv $i .config
-  Arch=`head -1 .config | cut -b 3-`
-  make ARCH=$Arch listnewconfig | grep -E '^CONFIG_' >.newoptions || true
 %if %{listnewconfig_fail}
-  if [ -s .newoptions ]; then
+	OPTS="$OPTS -n"
-    cat .newoptions
-    exit 1
-  fi
 %endif
-  rm -f .newoptions
-  make ARCH=$Arch oldnoconfig
-  echo "# $Arch" > configs/$i
-  cat .config >> configs/$i
 %if %{configmismatch_fail}
-  CheckConfigs configs/$i temp-$i
+	OPTS="$OPTS -c"
 %endif
-  rm temp-$i
+./process_configs.sh $OPTS kernel %{rpmversion}
-done
+
 # end of kernel config
 %endif
 
+cd ..
+# End of Configs stuff
+
 # get rid of unwanted files resulting from patch fuzz
 find . \( -name "*.orig" -o -name "*~" \) -exec rm -f {} \; >/dev/null
 

process_configs.sh

@@ -0,0 +1 @@
+configs/process_configs.sh