diff --git a/kernel.spec b/kernel.spec
index 101240c54..83064524d 100644
--- a/kernel.spec
+++ b/kernel.spec
@@ -1368,6 +1368,12 @@ pathfix.py -i "%{__python3} %{py3_shbang_opts}" -p -n \
 	Documentation \
 	scripts/gen_compile_commands.py
 
+# only deal with configs if we are going to build for the arch
+%ifnarch %nobuildarches
+
+if [ -L configs ]; then
+	rm -f configs
+fi
 # Deal with configs stuff
 mkdir configs
 cd configs
@@ -1390,9 +1396,6 @@ do
 done
 %endif
 
-# only deal with configs if we are going to build for the arch
-%ifnarch %nobuildarches
-
 %if !%{debugbuildsenabled}
 rm -f kernel-%{version}-*debug.config
 %endif