diff --git a/libsemanage.spec b/libsemanage.spec
index 5e9d799..7ec3e99 100644
--- a/libsemanage.spec
+++ b/libsemanage.spec
@@ -45,8 +45,9 @@ needed for developing applications that manipulate binary policies.
 %build
 make clean
 # Don't want to rebuild
+make CFLAGS="%{optflags}" all
 touch src/semanage_wrap.c
-make CFLAGS="%{optflags}" all pywrap
+make CFLAGS="%{optflags}" pywrap
 
 %install
 rm -rf ${RPM_BUILD_ROOT}