diff --git a/macros b/macros
index 1912bb7..236949e 100644
--- a/macros
+++ b/macros
@@ -380,10 +380,12 @@ for k,_ in pairs(stripped_flags) do print(k .. " ") end
 # to verify they have real sections/symbols after LTO stripping.  That
 # way we can detect installing an unusable .o/.a file.  This is on the TODO
 # list for F34.
-%_gcc_lto_cflags	-flto=auto -ffat-lto-objects
+# LTO is slow on riscv64, we want to prioritize build speeds right starting
+# Fedora 40.
+%_gcc_lto_cflags	%{nil}
 # riscv64 does not have ld.gold support needed for LTO with Clang.
 %_clang_lto_cflags	%{nil}
-%_lto_cflags		%{expand:%%{_%{toolchain}_lto_cflags}}
+%_lto_cflags		%{nil}
 
 # Default fortification level.
 #   "%define _fortify_level 2" to downgrade and
diff --git a/redhat-rpm-config.spec b/redhat-rpm-config.spec
index b1ed6d9..7d5864b 100644
--- a/redhat-rpm-config.spec
+++ b/redhat-rpm-config.spec
@@ -9,7 +9,7 @@
 Summary: Red Hat specific rpm configuration files
 Name: redhat-rpm-config
 Version: %{baserelease}
-Release: 1.0.riscv64%{?dist}
+Release: 1.1.riscv64%{?dist}
 # config.guess, config.sub are GPL-3.0-or-later WITH Autoconf-exception-generic
 License: GPL-1.0-or-later AND GPL-2.0-or-later AND GPL-3.0-or-later WITH Autoconf-exception-generic AND Boehm-GC
 URL: https://src.fedoraproject.org/rpms/redhat-rpm-config
@@ -255,6 +255,10 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora common.lua
 %doc buildflags.md
 
 %changelog
+* Thu Nov 30 2023 David Abdurachmanov <davidlt@rivosinc.com> - 271-1.1.riscv64
+- Set all LTO macros (_lto_cflags, _gcc_lto_cflags, _clang_lto_cflags) to nil
+  as we want to increase build times for packages.
+
 * Thu Nov 30 2023 David Abdurachmanov <davidlt@rivosinc.com> - 271-1.0.riscv64
 - Add riscv64 to %%gap_arches, %%java_arches, and %%nodejs_arches
 - Set %%_clang_lto_cflags to %%nil as ld.gold is not implemented for riscv64