Switch to unbundled wasi-libc on Fedora

Also, use emmalloc instead of CC0 dlmalloc when bundling wasi-libc.
2023-09-29 14:56:44 -07:00 · 2023-09-29 14:56:44 -07:00 · fbe7f8404b
parent 938db5293e
commit fbe7f8404b
6 changed files with 264 additions and 10 deletions
--- a/.gitignore
+++ b/.gitignore
@ -425,3 +425,4 @@
 /rustc-1.72.0-src.tar.xz
 /wasi-libc-7018e24d8fe248596819d2e884761676f3542a04.tar.gz
 /rustc-1.72.1-src.tar.xz
+/wasi-libc-bd950eb128bff337153de217b11270f948d04bb4.tar.gz
--- a/0001-bootstrap-allow-disabling-target-self-contained.patch
+++ b/0001-bootstrap-allow-disabling-target-self-contained.patch
@ -0,0 +1,102 @@
+From 19c37083cdae94105f6429350dd0b6617e3c2d56 Mon Sep 17 00:00:00 2001
+From: Josh Stone <jistone@redhat.com>
+Date: Thu, 28 Sep 2023 18:14:28 -0700
+Subject: [PATCH 1/2] bootstrap: allow disabling target self-contained
+
+---
+ config.example.toml      | 5 +++++
+ src/bootstrap/compile.rs | 4 ++++
+ src/bootstrap/config.rs  | 8 ++++++++
+ src/bootstrap/lib.rs     | 5 +++++
+ 4 files changed, 22 insertions(+)
+
+diff --git a/config.example.toml b/config.example.toml
+index 0c65b25fe138..6a0f1c01c328 100644
+--- a/config.example.toml
+++ b/config.example.toml
+@@ -789,6 +789,11 @@ changelog-seen = 2
+ # target triples containing `-none`, `nvptx`, `switch`, or `-uefi`.
+ #no-std = <platform-specific> (bool)
+ 
+# Copy libc and CRT objects into the target lib/self-contained/ directory.
+# Enabled by default on `musl`, `wasi`, and `windows-gnu` targets. Other
+# targets may ignore this setting if they have nothing to be contained.
+#self-contained = <platform-specific> (bool)
+
+ # =============================================================================
+ # Distribution options
+ #
+diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
+index 14c3ef79a78f..7adbf091acbb 100644
+--- a/src/bootstrap/compile.rs
+++ b/src/bootstrap/compile.rs
+@@ -263,6 +263,10 @@ fn copy_self_contained_objects(
+     compiler: &Compiler,
+     target: TargetSelection,
+ ) -> Vec<(PathBuf, DependencyType)> {
+    if builder.self_contained(target) != Some(true) {
+        return vec![];
+    }
+
+     let libdir_self_contained = builder.sysroot_libdir(*compiler, target).join("self-contained");
+     t!(fs::create_dir_all(&libdir_self_contained));
+     let mut target_deps = vec![];
+diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
+index fe932fd6bd30..a626badc3e8a 100644
+--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
+@@ -541,6 +541,7 @@ pub struct Target {
+     pub wasi_root: Option<PathBuf>,
+     pub qemu_rootfs: Option<PathBuf>,
+     pub no_std: bool,
+    pub self_contained: bool,
+ }
+ 
+ impl Target {
+@@ -553,6 +554,9 @@ pub fn from_triple(triple: &str) -> Self {
+         {
+             target.no_std = true;
+         }
+        if triple.contains("-musl") || triple.contains("-wasi") || triple.contains("-windows-gnu") {
+            target.self_contained = true;
+        }
+         target
+     }
+ }
+@@ -999,6 +1003,7 @@ struct TomlTarget {
+         wasi_root: Option<String> = "wasi-root",
+         qemu_rootfs: Option<String> = "qemu-rootfs",
+         no_std: Option<bool> = "no-std",
+        self_contained: Option<bool> = "self-contained",
+     }
+ }
+ 
+@@ -1524,6 +1529,9 @@ fn get_table(option: &str) -> Result<TomlConfig, toml::de::Error> {
+                 if let Some(s) = cfg.no_std {
+                     target.no_std = s;
+                 }
+                if let Some(s) = cfg.self_contained {
+                    target.self_contained = s;
+                }
+                 target.cc = cfg.cc.map(PathBuf::from).or_else(|| {
+                     target.ndk.as_ref().map(|ndk| ndk_compiler(Language::C, &triple, ndk))
+                 });
+diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
+index 0a7aff62257a..9b7c18a3353f 100644
+--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
+@@ -1285,6 +1285,11 @@ fn no_std(&self, target: TargetSelection) -> Option<bool> {
+         self.config.target_config.get(&target).map(|t| t.no_std)
+     }
+ 
+    /// Returns `true` if this is a self-contained `target`, if defined
+    fn self_contained(&self, target: TargetSelection) -> Option<bool> {
+        self.config.target_config.get(&target).map(|t| t.self_contained)
+    }
+
+     /// Returns `true` if the target will be tested using the `remote-test-client`
+     /// and `remote-test-server` binaries.
+     fn remote_tested(&self, target: TargetSelection) -> bool {
+-- 
+2.41.0
+
--- a/0001-wasi-round-up-the-size-for-aligned_alloc.patch
+++ b/0001-wasi-round-up-the-size-for-aligned_alloc.patch
@ -0,0 +1,33 @@
+From 1c6d867d78475fd8c6274e2b64ebb27735b6cabf Mon Sep 17 00:00:00 2001
+From: Josh Stone <jistone@redhat.com>
+Date: Sat, 26 Aug 2023 11:50:16 -0700
+Subject: [PATCH] wasi: round up the size for `aligned_alloc`
+
+C11 `aligned_alloc` requires that the size be a multiple of the
+alignment. This is enforced in the wasi-libc emmalloc implementation,
+which always returns NULL if the size is not a multiple.
+(The default `MALLOC_IMPL=dlmalloc` does not currently check this.)
+---
+ library/std/src/sys/unix/alloc.rs | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/library/std/src/sys/unix/alloc.rs b/library/std/src/sys/unix/alloc.rs
+index 8604b53983d6..af0089978ecb 100644
+--- a/library/std/src/sys/unix/alloc.rs
+++ b/library/std/src/sys/unix/alloc.rs
+@@ -86,7 +86,11 @@ unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 {
+     } else if #[cfg(target_os = "wasi")] {
+         #[inline]
+         unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 {
+-            libc::aligned_alloc(layout.align(), layout.size()) as *mut u8
+            // C11 aligned_alloc requires that the size be a multiple of the alignment.
+            // Layout already checks that the size rounded up doesn't overflow isize::MAX.
+            let align = layout.align();
+            let size = layout.size().next_multiple_of(align);
+            libc::aligned_alloc(align, size) as *mut u8
+         }
+     } else {
+         #[inline]
+-- 
+2.41.0
+
--- a/0002-set-an-external-library-path-for-wasm32-wasi.patch
+++ b/0002-set-an-external-library-path-for-wasm32-wasi.patch
@ -0,0 +1,78 @@
+From 3016b2b7052d8b01d50c3a3c6591aeb99d918ca3 Mon Sep 17 00:00:00 2001
+From: Josh Stone <jistone@redhat.com>
+Date: Thu, 28 Sep 2023 18:18:16 -0700
+Subject: [PATCH 2/2] set an external library path for wasm32-wasi
+
+---
+ compiler/rustc_codegen_ssa/src/back/link.rs   | 9 +++++++++
+ compiler/rustc_target/src/spec/mod.rs         | 2 ++
+ compiler/rustc_target/src/spec/wasm32_wasi.rs | 6 +++++-
+ 3 files changed, 16 insertions(+), 1 deletion(-)
+
+diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
+index b603a8787460..40d878b64479 100644
+--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
+@@ -1475,6 +1475,12 @@ fn get_object_file_path(sess: &Session, name: &str, self_contained: bool) -> Pat
+             return file_path;
+         }
+     }
+    if let Some(lib_path) = &sess.target.options.external_lib_path {
+        let file_path = Path::new(lib_path.as_ref()).join(name);
+        if file_path.exists() {
+            return file_path;
+        }
+    }
+     for search_path in fs.search_paths() {
+         let file_path = search_path.dir.join(name);
+         if file_path.exists() {
+@@ -1967,6 +1973,9 @@ fn add_library_search_dirs(cmd: &mut dyn Linker, sess: &Session, self_contained:
+         let lib_path = sess.target_filesearch(PathKind::All).get_self_contained_lib_path();
+         cmd.include_path(&fix_windows_verbatim_for_gcc(&lib_path));
+     }
+    if let Some(lib_path) = &sess.target.options.external_lib_path {
+        cmd.include_path(Path::new(lib_path.as_ref()));
+    }
+ }
+ 
+ /// Add options making relocation sections in the produced ELF files read-only
+diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
+index 2365dfaf1af8..35f3a686cf67 100644
+--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
+@@ -1653,6 +1653,7 @@ pub struct TargetOptions {
+     /// Objects to link before and after all other object code.
+     pub pre_link_objects: CrtObjects,
+     pub post_link_objects: CrtObjects,
+    pub external_lib_path: Option<StaticCow<str>>,
+     /// Same as `(pre|post)_link_objects`, but when self-contained linking mode is enabled.
+     pub pre_link_objects_self_contained: CrtObjects,
+     pub post_link_objects_self_contained: CrtObjects,
+@@ -2124,6 +2125,7 @@ fn default() -> TargetOptions {
+             relro_level: RelroLevel::None,
+             pre_link_objects: Default::default(),
+             post_link_objects: Default::default(),
+            external_lib_path: None,
+             pre_link_objects_self_contained: Default::default(),
+             post_link_objects_self_contained: Default::default(),
+             link_self_contained: LinkSelfContainedDefault::False,
+diff --git a/compiler/rustc_target/src/spec/wasm32_wasi.rs b/compiler/rustc_target/src/spec/wasm32_wasi.rs
+index a0476d542e64..ad7160bf5fcd 100644
+--- a/compiler/rustc_target/src/spec/wasm32_wasi.rs
+++ b/compiler/rustc_target/src/spec/wasm32_wasi.rs
+@@ -85,7 +85,11 @@ pub fn target() -> Target {
+     options.post_link_objects_self_contained = crt_objects::post_wasi_self_contained();
+ 
+     // FIXME: Figure out cases in which WASM needs to link with a native toolchain.
+-    options.link_self_contained = LinkSelfContainedDefault::True;
+    options.link_self_contained = LinkSelfContainedDefault::False;
+
+    options.pre_link_objects = options.pre_link_objects_self_contained.clone();
+    options.post_link_objects = options.post_link_objects_self_contained.clone();
+    options.external_lib_path = Some("/usr/wasm32-wasi/lib/wasm32-wasi".into());
+ 
+     // Right now this is a bit of a workaround but we're currently saying that
+     // the target by default has a static crt which we're taking as a signal
+-- 
+2.41.0
+
--- a/rust.spec
+++ b/rust.spec
@ -43,10 +43,15 @@
 # (updated per https://github.com/rust-lang/rust/pull/96907)
 %global wasi_libc_url https://github.com/WebAssembly/wasi-libc
 #global wasi_libc_ref wasi-sdk-20
-%global wasi_libc_ref 7018e24d8fe248596819d2e884761676f3542a04
+%global wasi_libc_ref bd950eb128bff337153de217b11270f948d04bb4
 %global wasi_libc_name wasi-libc-%{wasi_libc_ref}
 %global wasi_libc_source %{wasi_libc_url}/archive/%{wasi_libc_ref}/%{wasi_libc_name}.tar.gz
 %global wasi_libc_dir %{_builddir}/%{wasi_libc_name}
+%if 0%{?fedora}
+%bcond_with bundled_wasi_libc
+%else
+%bcond_without bundled_wasi_libc
+%endif

 # Using llvm-static may be helpful as an opt-in, e.g. to aid LLVM rebases.
 %bcond_with llvm_static
@ -118,17 +123,27 @@ Patch2:         rustc-1.70.0-rust-gdb-substitute-path.patch
 # TODO: upstream this ability into the actual build configuration
 Patch3:         0001-Let-environment-variables-override-some-default-CPUs.patch

+# Override the default self-contained system libraries
+# TODO: the first can probably be upstreamed, but the second is hard-coded,
+# and we're only applying that if not with bundled_wasi_libc.
+Patch4:         0001-bootstrap-allow-disabling-target-self-contained.patch
+Patch5:         0002-set-an-external-library-path-for-wasm32-wasi.patch
+
 # Enable the profiler runtime for native hosts
 # https://github.com/rust-lang/rust/pull/114069
-Patch4:         0001-Allow-using-external-builds-of-the-compiler-rt-profi.patch
+Patch6:         0001-Allow-using-external-builds-of-the-compiler-rt-profi.patch

 # Fix --no-fail-fast
 # https://github.com/rust-lang/rust/pull/113214
-Patch5:         0001-Don-t-fail-early-if-try_run-returns-an-error.patch
+Patch7:         0001-Don-t-fail-early-if-try_run-returns-an-error.patch

 # The dist-src tarball doesn't include .github/
 # https://github.com/rust-lang/rust/pull/115109
-Patch6:         0001-Skip-ExpandYamlAnchors-when-the-config-is-missing.patch
+Patch8:         0001-Skip-ExpandYamlAnchors-when-the-config-is-missing.patch
+
+# wasi: round up the size for aligned_alloc
+# https://github.com/rust-lang/rust/pull/115254
+Patch9:         0001-wasi-round-up-the-size-for-aligned_alloc.patch

 ### RHEL-specific patches below ###

@ -336,7 +351,11 @@ BuildRequires:  mingw64-winpthreads-static
 %endif

 %if %defined wasm_targets
+%if %with bundled_wasi_libc
 BuildRequires:  clang
+%else
+BuildRequires:  wasi-libc-static
+%endif
 BuildRequires:  lld
 # brp-strip-static-archive breaks the archive index for wasm
 %global __os_install_post \
@ -413,7 +432,11 @@ BuildArch:      noarch
 %if %target_enabled wasm32-wasi
 %target_package wasm32-wasi
 Requires:       lld >= 8.0
+%if %with bundled_wasi_libc
 Provides:       bundled(wasi-libc)
+%else
+Requires:       wasi-libc-static
+%endif
 BuildArch:      noarch
 %target_description wasm32-wasi WebAssembly
 %endif
@ -595,8 +618,9 @@ test -f '%{local_rust_root}/bin/cargo'
 test -f '%{local_rust_root}/bin/rustc'
 %endif

-%if %defined wasm_targets
+%if %{defined wasm_targets} && %{with bundled_wasi_libc}
 %setup -q -n %{wasi_libc_name} -T -b 1
+rm -rf %{wasi_libc_dir}/dlmalloc/
 %endif

 %setup -q -n %{rustc_package}
@ -605,8 +629,13 @@ test -f '%{local_rust_root}/bin/rustc'
 %patch -P2 -p1
 %patch -P3 -p1
 %patch -P4 -p1
+%if %without bundled_wasi_libc
 %patch -P5 -p1
+%endif
 %patch -P6 -p1
+%patch -P7 -p1
+%patch -P8 -p1
+%patch -P9 -p1

 %if %with disabled_libssh2
 %patch -P100 -p1
@ -733,16 +762,25 @@ fi
  --set target.i686-pc-windows-gnu.cc=%{mingw32_cc}
  --set target.i686-pc-windows-gnu.ar=%{mingw32_ar}
  --set target.i686-pc-windows-gnu.ranlib=%{mingw32_ranlib}
+  --set target.i686-pc-windows-gnu.self-contained=false
  --set target.x86_64-pc-windows-gnu.linker=%{mingw64_cc}
  --set target.x86_64-pc-windows-gnu.cc=%{mingw64_cc}
  --set target.x86_64-pc-windows-gnu.ar=%{mingw64_ar}
  --set target.x86_64-pc-windows-gnu.ranlib=%{mingw64_ranlib}
+  --set target.x86_64-pc-windows-gnu.self-contained=false
 }
 %endif

 %if %defined wasm_targets
-%make_build --quiet -C %{wasi_libc_dir} CC=clang AR=llvm-ar NM=llvm-nm
+%if %with bundled_wasi_libc
+%make_build --quiet -C %{wasi_libc_dir} MALLOC_IMPL=emmalloc CC=clang AR=llvm-ar NM=llvm-nm
 %define wasm_target_config --set target.wasm32-wasi.wasi-root=%{wasi_libc_dir}/sysroot
+%else
+%define wasm_target_config %{shrink:
+  --set target.wasm32-wasi.wasi-root=%{_prefix}/wasm32-wasi
+  --set target.wasm32-wasi.self-contained=false
+}
+%endif
 %endif

 %if 0%{?fedora} || 0%{?rhel} >= 8
@ -969,7 +1007,6 @@ rm -rf "./build/%{rust_triple}/stage2-tools/%{rust_triple}/cit/"
 %{rustlibdir}/i686-pc-windows-gnu/lib/rs*.o
 %exclude %{rustlibdir}/i686-pc-windows-gnu/lib/*.dll
 %exclude %{rustlibdir}/i686-pc-windows-gnu/lib/*.dll.a
-%exclude %{rustlibdir}/i686-pc-windows-gnu/lib/self-contained
 %endif

 %if %target_enabled x86_64-pc-windows-gnu
@ -977,7 +1014,6 @@ rm -rf "./build/%{rust_triple}/stage2-tools/%{rust_triple}/cit/"
 %{rustlibdir}/x86_64-pc-windows-gnu/lib/rs*.o
 %exclude %{rustlibdir}/x86_64-pc-windows-gnu/lib/*.dll
 %exclude %{rustlibdir}/x86_64-pc-windows-gnu/lib/*.dll.a
-%exclude %{rustlibdir}/x86_64-pc-windows-gnu/lib/self-contained
 %endif

 %if %target_enabled wasm32-unknown-unknown
@ -986,10 +1022,12 @@ rm -rf "./build/%{rust_triple}/stage2-tools/%{rust_triple}/cit/"

 %if %target_enabled wasm32-wasi
 %target_files wasm32-wasi
+%if %with bundled_wasi_libc
 %dir %{rustlibdir}/wasm32-wasi/lib/self-contained
 %{rustlibdir}/wasm32-wasi/lib/self-contained/crt*.o
 %{rustlibdir}/wasm32-wasi/lib/self-contained/libc.a
 %endif
+%endif

 %if %target_enabled x86_64-unknown-none
 %target_files x86_64-unknown-none
@ -1074,8 +1112,10 @@ rm -rf "./build/%{rust_triple}/stage2-tools/%{rust_triple}/cit/"


 %changelog
-* Wed Sep 27 2023 Josh Stone <jistone@redhat.com> - 1.72.1-3
+* Fri Sep 29 2023 Josh Stone <jistone@redhat.com> - 1.72.1-3
 - Fix the profiler runtime with compiler-rt-17
+- Switch to unbundled wasi-libc on Fedora
+- Use emmalloc instead of CC0 dlmalloc when bundling wasi-libc

 * Mon Sep 25 2023 Josh Stone <jistone@redhat.com> - 1.72.1-2
 - Fix LLVM dependency for ELN
--- a/2
+++ b/2
@ -1,2 +1,2 @@
 SHA512 (rustc-1.72.1-src.tar.xz) = 08232b5bf36f82a995d67f3d03d5e35b7d8914d31fb4491d4c37b72a830bc438e9d18d9e138d398b1b6ae4aa09f7f8e1e9b68da6273ab74bdae4c6123586a21b
-SHA512 (wasi-libc-7018e24d8fe248596819d2e884761676f3542a04.tar.gz) = a2a4a952c3d9795792be8f055387057befaebe0675ad2464a478cb1f2c45d65f233e0ee4c4dbcaa137bf9649882ff6c6acf2f2bec07b2ad89f63ff980d972e6b
+SHA512 (wasi-libc-bd950eb128bff337153de217b11270f948d04bb4.tar.gz) = 01e5cc3ebdab239f57816ff80f939fd87a5491a28951daf74b3310b118b4820c098ac9417771c9c6af55ca91d2cabe6498975ab9db4914aba754d87067cd1066