rpms/rpm
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Cap number of threads on 32bit platforms (#1729382)

Browse Source 
Also drop %_lto_cflags macro (reverted upstream) before people
start adapting its use.
This commit is contained in:
Panu Matilainen 2019-08-23 13:52:16 +03:00
parent c9c421374c
commit d678a6a69f
4 changed files with 153 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
0001-Drop-_lto_cflags-macro-afterall.patch Normal file
View File

@ -0,0 +1,50 @@
From 7faf8eda1358f8a877b9b3d6e1197b814e80b50b Mon Sep 17 00:00:00 2001
Message-Id: <7faf8eda1358f8a877b9b3d6e1197b814e80b50b.1566557361.git.pmatilai@redhat.com>
From: Panu Matilainen <pmatilai@redhat.com>
Date: Tue, 13 Aug 2019 16:26:31 +0300
Subject: [PATCH] Drop %_lto_cflags macro afterall
This was only added in commit 2bb7b0cf066c97a9d92eb0bf59618896000cb29d,
but turns out that this kind of usage is bad for build reproducability
because the system-specific CPU count gets recorded RPMTAG_OPTFLAGS
and the resulting binaries too (depending on gcc flags).
In addition, gcc upstream has decided to make -flto default to
autodetected parallelism. Since -flto can be overridden with
by simply appending -fno-lto for the packages that need to disable it,
there's no practical need for us to provide such a macro for disabling
either.
---
macros.in | 2 +-
platform.in | 3 ---
2 files changed, 1 insertion(+), 4 deletions(-)
diff --git a/macros.in b/macros.in
index 633d5ca6e..b7da4ea2f 100644
--- a/macros.in
+++ b/macros.in
@@ -1025,7 +1025,7 @@ package or when debugging this package.\
%build_fflags %{optflags} %{?_fmoddir:-I%{_fmoddir}}
# Link editor flags. This is usually called LDFLAGS in makefiles.
-#%build_ldflags -Wl,-z,relro %{?_lto_cflags}
+#%build_ldflags -Wl,-z,relro
# Expands to shell code to seot the compiler/linker environment
# variables CFLAGS, CXXFLAGS, FFLAGS, FCFLAGS, LDFLAGS if they have
diff --git a/platform.in b/platform.in
index e1efc42b0..db6d2382f 100644
--- a/platform.in
+++ b/platform.in
@@ -59,9 +59,6 @@
%_smp_mflags -j%{_smp_build_ncpus}
-# Enable LTO optimization with a maximal parallelism
-%_lto_cflags -flto=%{_smp_build_ncpus}
-
#==============================================================================
# ---- Build policy macros.
#
--
2.21.0

32
0001-Fix-build-code-thread-cap-logic-for-unlimited-CPUs.patch Normal file
View File

@ -0,0 +1,32 @@
From 3ba0136000c986c692baab7b112aba163173d2d3 Mon Sep 17 00:00:00 2001
Message-Id: <3ba0136000c986c692baab7b112aba163173d2d3.1566556488.git.pmatilai@redhat.com>
From: Panu Matilainen <pmatilai@redhat.com>
Date: Fri, 23 Aug 2019 11:09:40 +0300
Subject: [PATCH 1/2] Fix build code thread cap logic for unlimited CPUs
If there's no clear cap set from rpm configuration, just let OMP do
its own thing (ie use all available CPUs) instead of artificially
limiting to 1.
---
build/parseSpec.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/build/parseSpec.c b/build/parseSpec.c
index 055bdf012..61858aabd 100644
--- a/build/parseSpec.c
+++ b/build/parseSpec.c
@@ -1036,9 +1036,8 @@ static rpmSpec parseSpec(const char *specFile, rpmSpecFlags flags,
#ifdef ENABLE_OPENMP
/* Set number of OMP threads centrally */
int ncpus = rpmExpandNumeric("%{?_smp_build_ncpus}");
- if (ncpus <= 0)
- ncpus = 1;
- omp_set_num_threads(ncpus);
+ if (ncpus > 0) {
+ omp_set_num_threads(ncpus);
#endif
if (spec->clean == NULL) {
--
2.21.0

62
0002-Cap-number-of-threads-on-32bit-platforms-add-a-tunab.patch Normal file
View File

@ -0,0 +1,62 @@
From bcaa30872087a1002648653358ab9514f829de9a Mon Sep 17 00:00:00 2001
Message-Id: <bcaa30872087a1002648653358ab9514f829de9a.1566556488.git.pmatilai@redhat.com>
In-Reply-To: <3ba0136000c986c692baab7b112aba163173d2d3.1566556488.git.pmatilai@redhat.com>
References: <3ba0136000c986c692baab7b112aba163173d2d3.1566556488.git.pmatilai@redhat.com>
From: Panu Matilainen <pmatilai@redhat.com>
Date: Fri, 23 Aug 2019 11:17:21 +0300
Subject: [PATCH 2/2] Cap number of threads on 32bit platforms, add a tunable
(RhBug:1729382)
On 32bit plaforms, address space is easily exhausted with multiple
threads, causing arbitrary builds failure regressions (RhBug:1729382).
Simply cap the number of threads to maximum of four on any 32bit
platform to play it safe. It's still three more than we were able to
use on older releases...
In addition, introduce a separate tunable for tweaking the maximum
number of threads just in case, defaulting to max CPUs available.
---
build/parseSpec.c | 10 +++++++++-
platform.in | 3 +++
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/build/parseSpec.c b/build/parseSpec.c
index 61858aabd..d987bdc7b 100644
--- a/build/parseSpec.c
+++ b/build/parseSpec.c
@@ -1035,9 +1035,17 @@ static rpmSpec parseSpec(const char *specFile, rpmSpecFlags flags,
#ifdef ENABLE_OPENMP
/* Set number of OMP threads centrally */
- int ncpus = rpmExpandNumeric("%{?_smp_build_ncpus}");
+ int ncpus = rpmExpandNumeric("%{?_smp_nthreads_max}");
+#if __WORDSIZE == 32
+ /* On 32bit platforms, address space shortage is an issue. Play safe. */
+ if (ncpus <= 0 || ncpus > 4) {
+ ncpus = 4;
+ rpmlog(RPMLOG_DEBUG,
+ "limiting number of threads to %d due to platform\n", ncpus);
+ }
if (ncpus > 0) {
omp_set_num_threads(ncpus);
+#endif
#endif
if (spec->clean == NULL) {
diff --git a/platform.in b/platform.in
index db6d2382f..c70f44193 100644
--- a/platform.in
+++ b/platform.in
@@ -59,6 +59,9 @@
%_smp_mflags -j%{_smp_build_ncpus}
+# Maximum number of CPU's to use for threads
+%_smp_nthreads_max %{_smp_build_ncpus}
+
#==============================================================================
# ---- Build policy macros.
#
--
2.21.0

10
rpm.spec
View File

@ -21,7 +21,7 @@
%global rpmver 4.15.0
%global snapver beta
%global rel 3
%global rel 4
%global srcver %{version}%{?snapver:-%{snapver}}
%global srcdir %{?snapver:testing}%{!?snapver:%{name}-%(echo %{version} | cut -d'.' -f1-2).x}
@ -57,8 +57,12 @@ Patch101: 0001-Do-not-set-RPMTAG_BUILDTIME-to-SOURCE_DATE_EPOCH-whe.patch
Patch110: 0001-Support-running-rpmfcExec-without-any-piped-input-ou.patch
Patch111: 0002-Restore-strict-order-of-build-scriptlet-stdout-stder.patch
Patch112: 0003-Drop-the-no-longer-needed-rpmfcExec-output-duplicati.patch
Patch113: 0001-Drop-_lto_cflags-macro-afterall.patch
# These are not yet upstream
Patch200: 0001-Fix-build-code-thread-cap-logic-for-unlimited-CPUs.patch
Patch201: 0002-Cap-number-of-threads-on-32bit-platforms-add-a-tunab.patch
Patch906: rpm-4.7.1-geode-i686.patch
# Probably to be upstreamed in slightly different form
Patch907: rpm-4.15.x-ldflags.patch
@ -543,6 +547,10 @@ make check || (cat tests/rpmtests.log; exit 0)
%doc doc/librpm/html/*
%changelog
* Fri Aug 23 2019 Panu Matilainen <pmatilai@redhat.com> - 4.15.0-0.beta.4
- Cap number of threads on 32bit platforms (#1729382)
- Drop %%_lto_cflags macro (reverted upstream)
* Fri Aug 23 2019 Panu Matilainen <pmatilai@redhat.com> - 4.15.0-0.beta.3
- Restore strict order of build scriptlet stdout/stderr output