235 lines
8.6 KiB
Diff
235 lines
8.6 KiB
Diff
diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go
|
|
index fef8add..30c3a67 100644
|
|
--- a/src/runtime/cgocall.go
|
|
+++ b/src/runtime/cgocall.go
|
|
@@ -568,7 +568,7 @@ func cgoIsGoPointer(p unsafe.Pointer) bool {
|
|
return false
|
|
}
|
|
|
|
- if cgoInRange(p, mheap_.arena_start, mheap_.arena_used) {
|
|
+ if inHeapOrStack(uintptr(p)) {
|
|
return true
|
|
}
|
|
|
|
diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
|
|
index b520c68..2055da5 100644
|
|
--- a/src/runtime/malloc.go
|
|
+++ b/src/runtime/malloc.go
|
|
@@ -173,7 +173,7 @@ const (
|
|
// Page number (address>>pageShift)
|
|
type pageID uintptr
|
|
|
|
-const _MaxArena32 = 2 << 30
|
|
+const _MaxArena32 = 1<<32 - 1
|
|
|
|
// OS-defined helpers:
|
|
//
|
|
@@ -230,7 +230,7 @@ func mallocinit() {
|
|
|
|
// Set up the allocation arena, a contiguous area of memory where
|
|
// allocated data will be found. The arena begins with a bitmap large
|
|
- // enough to hold 4 bits per allocated word.
|
|
+ // enough to hold 2 bits per allocated word.
|
|
if sys.PtrSize == 8 && (limit == 0 || limit > 1<<30) {
|
|
// On a 64-bit machine, allocate from a single contiguous reservation.
|
|
// 512 GB (MaxMem) should be big enough for now.
|
|
@@ -262,7 +262,7 @@ func mallocinit() {
|
|
// translation buffers, the user address space is limited to 39 bits
|
|
// On darwin/arm64, the address space is even smaller.
|
|
arenaSize := round(_MaxMem, _PageSize)
|
|
- bitmapSize = arenaSize / (sys.PtrSize * 8 / 4)
|
|
+ bitmapSize = arenaSize / (sys.PtrSize * 8 / 2)
|
|
spansSize = arenaSize / _PageSize * sys.PtrSize
|
|
spansSize = round(spansSize, _PageSize)
|
|
for i := 0; i <= 0x7f; i++ {
|
|
@@ -287,7 +287,7 @@ func mallocinit() {
|
|
// with a giant virtual address space reservation.
|
|
// Instead we map the memory information bitmap
|
|
// immediately after the data segment, large enough
|
|
- // to handle another 2GB of mappings (256 MB),
|
|
+ // to handle the entire 4GB of mappings (256 MB),
|
|
// along with a reservation for an initial arena.
|
|
// When that gets used up, we'll start asking the kernel
|
|
// for any memory anywhere and hope it's in the 2GB
|
|
@@ -304,15 +304,18 @@ func mallocinit() {
|
|
// If we fail to allocate, try again with a smaller arena.
|
|
// This is necessary on Android L where we share a process
|
|
// with ART, which reserves virtual memory aggressively.
|
|
+ // In the worst case, fall back to a 0-sized initial arena,
|
|
+ // in the hope that subsequent reservations will succeed.
|
|
arenaSizes := []uintptr{
|
|
512 << 20,
|
|
256 << 20,
|
|
128 << 20,
|
|
+ 0,
|
|
}
|
|
|
|
for _, arenaSize := range arenaSizes {
|
|
- bitmapSize = _MaxArena32 / (sys.PtrSize * 8 / 4)
|
|
- spansSize = _MaxArena32 / _PageSize * sys.PtrSize
|
|
+ bitmapSize = (_MaxArena32 + 1) / (sys.PtrSize * 8 / 2)
|
|
+ spansSize = (_MaxArena32 + 1) / _PageSize * sys.PtrSize
|
|
if limit > 0 && arenaSize+bitmapSize+spansSize > limit {
|
|
bitmapSize = (limit / 9) &^ ((1 << _PageShift) - 1)
|
|
arenaSize = bitmapSize * 8
|
|
@@ -347,10 +350,16 @@ func mallocinit() {
|
|
p1 := round(p, _PageSize)
|
|
|
|
mheap_.spans = (**mspan)(unsafe.Pointer(p1))
|
|
- mheap_.bitmap = p1 + spansSize
|
|
- mheap_.arena_start = p1 + (spansSize + bitmapSize)
|
|
- mheap_.arena_used = mheap_.arena_start
|
|
+ mheap_.bitmap = p1 + spansSize + bitmapSize
|
|
+ if sys.PtrSize == 4 {
|
|
+ // Set arena_start such that we can accept memory
|
|
+ // reservations located anywhere in the 4GB virtual space.
|
|
+ mheap_.arena_start = 0
|
|
+ } else {
|
|
+ mheap_.arena_start = p1 + (spansSize + bitmapSize)
|
|
+ }
|
|
mheap_.arena_end = p + pSize
|
|
+ mheap_.arena_used = p1 + (spansSize + bitmapSize)
|
|
mheap_.arena_reserved = reserved
|
|
|
|
if mheap_.arena_start&(_PageSize-1) != 0 {
|
|
@@ -364,36 +373,13 @@ func mallocinit() {
|
|
_g_.m.mcache = allocmcache()
|
|
}
|
|
|
|
-// sysReserveHigh reserves space somewhere high in the address space.
|
|
-// sysReserve doesn't actually reserve the full amount requested on
|
|
-// 64-bit systems, because of problems with ulimit. Instead it checks
|
|
-// that it can get the first 64 kB and assumes it can grab the rest as
|
|
-// needed. This doesn't work well with the "let the kernel pick an address"
|
|
-// mode, so don't do that. Pick a high address instead.
|
|
-func sysReserveHigh(n uintptr, reserved *bool) unsafe.Pointer {
|
|
- if sys.PtrSize == 4 {
|
|
- return sysReserve(nil, n, reserved)
|
|
- }
|
|
-
|
|
- for i := 0; i <= 0x7f; i++ {
|
|
- p := uintptr(i)<<40 | uintptrMask&(0x00c0<<32)
|
|
- *reserved = false
|
|
- p = uintptr(sysReserve(unsafe.Pointer(p), n, reserved))
|
|
- if p != 0 {
|
|
- return unsafe.Pointer(p)
|
|
- }
|
|
- }
|
|
-
|
|
- return sysReserve(nil, n, reserved)
|
|
-}
|
|
-
|
|
func (h *mheap) sysAlloc(n uintptr) unsafe.Pointer {
|
|
if n > h.arena_end-h.arena_used {
|
|
// We are in 32-bit mode, maybe we didn't use all possible address space yet.
|
|
// Reserve some more space.
|
|
p_size := round(n+_PageSize, 256<<20)
|
|
new_end := h.arena_end + p_size // Careful: can overflow
|
|
- if h.arena_end <= new_end && new_end <= h.arena_start+_MaxArena32 {
|
|
+ if h.arena_end <= new_end && new_end-h.arena_start-1 <= _MaxArena32 {
|
|
// TODO: It would be bad if part of the arena
|
|
// is reserved and part is not.
|
|
var reserved bool
|
|
@@ -404,7 +390,7 @@ func (h *mheap) sysAlloc(n uintptr) unsafe.Pointer {
|
|
if p == h.arena_end {
|
|
h.arena_end = new_end
|
|
h.arena_reserved = reserved
|
|
- } else if h.arena_start <= p && p+p_size <= h.arena_start+_MaxArena32 {
|
|
+ } else if h.arena_start <= p && p+p_size-h.arena_start-1 <= _MaxArena32 {
|
|
// Keep everything page-aligned.
|
|
// Our pages are bigger than hardware pages.
|
|
h.arena_end = p + p_size
|
|
@@ -441,23 +427,24 @@ func (h *mheap) sysAlloc(n uintptr) unsafe.Pointer {
|
|
}
|
|
|
|
// If using 64-bit, our reservation is all we have.
|
|
- if h.arena_end-h.arena_start >= _MaxArena32 {
|
|
+ if h.arena_end-h.arena_start > _MaxArena32 {
|
|
return nil
|
|
}
|
|
|
|
// On 32-bit, once the reservation is gone we can
|
|
// try to get memory at a location chosen by the OS
|
|
// and hope that it is in the range we allocated bitmap for.
|
|
+ // try to get memory at a location chosen by the OS.
|
|
p_size := round(n, _PageSize) + _PageSize
|
|
p := uintptr(sysAlloc(p_size, &memstats.heap_sys))
|
|
if p == 0 {
|
|
return nil
|
|
}
|
|
|
|
- if p < h.arena_start || uintptr(p)+p_size-h.arena_start >= _MaxArena32 {
|
|
+ if p < h.arena_start || p+p_size-h.arena_start > _MaxArena32 {
|
|
top := ^uintptr(0)
|
|
- if top-h.arena_start > _MaxArena32 {
|
|
- top = h.arena_start + _MaxArena32
|
|
+ if top-h.arena_start-1 > _MaxArena32 {
|
|
+ top = h.arena_start + _MaxArena32 + 1
|
|
}
|
|
print("runtime: memory allocated by OS (", hex(p), ") not in usable range [", hex(h.arena_start), ",", hex(top), ")\n")
|
|
sysFree(unsafe.Pointer(p), p_size, &memstats.heap_sys)
|
|
diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go
|
|
index 336d4d8..9a5d83b 100644
|
|
--- a/src/runtime/mbitmap.go
|
|
+++ b/src/runtime/mbitmap.go
|
|
@@ -145,7 +145,7 @@ func (h *mheap) mapBits(arena_used uintptr) {
|
|
return
|
|
}
|
|
|
|
- sysMap(unsafe.Pointer(h.arena_start-n), n-h.bitmap_mapped, h.arena_reserved, &memstats.gc_sys)
|
|
+ sysMap(unsafe.Pointer(h.bitmap-n), n-h.bitmap_mapped, h.arena_reserved, &memstats.gc_sys)
|
|
h.bitmap_mapped = n
|
|
}
|
|
|
|
@@ -166,7 +166,7 @@ type heapBits struct {
|
|
func heapBitsForAddr(addr uintptr) heapBits {
|
|
// 2 bits per work, 4 pairs per byte, and a mask is hard coded.
|
|
off := (addr - mheap_.arena_start) / sys.PtrSize
|
|
- return heapBits{(*uint8)(unsafe.Pointer(mheap_.arena_start - off/4 - 1)), uint32(off & 3)}
|
|
+ return heapBits{(*uint8)(unsafe.Pointer(mheap_.bitmap - off/4 - 1)), uint32(off & 3)}
|
|
}
|
|
|
|
// heapBitsForSpan returns the heapBits for the span base address base.
|
|
diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
|
|
index a153df0..1db6a49 100644
|
|
--- a/src/runtime/mheap.go
|
|
+++ b/src/runtime/mheap.go
|
|
@@ -46,7 +46,7 @@ type mheap struct {
|
|
nsmallfree [_NumSizeClasses]uint64 // number of frees for small objects (<=maxsmallsize)
|
|
|
|
// range of addresses we might see in the heap
|
|
- bitmap uintptr
|
|
+ bitmap uintptr // Points to one byte past the end of the bitmap
|
|
bitmap_mapped uintptr
|
|
arena_start uintptr
|
|
arena_used uintptr // always mHeap_Map{Bits,Spans} before updating
|
|
@@ -217,6 +217,28 @@ func inheap(b uintptr) bool {
|
|
return true
|
|
}
|
|
|
|
+// inHeapOrStack is a variant of inheap that returns true for pointers into stack spans.
|
|
+//go:nowritebarrier
|
|
+//go:nosplit
|
|
+func inHeapOrStack(b uintptr) bool {
|
|
+ if b == 0 || b < mheap_.arena_start || b >= mheap_.arena_used {
|
|
+ return false
|
|
+ }
|
|
+ // Not a beginning of a block, consult span table to find the block beginning.
|
|
+ s := h_spans[(b-mheap_.arena_start)>>_PageShift]
|
|
+ if s == nil || b < s.base() {
|
|
+ return false
|
|
+ }
|
|
+ switch s.state {
|
|
+ case mSpanInUse:
|
|
+ return b < s.limit
|
|
+ case _MSpanStack:
|
|
+ return b < s.base()+s.npages<<_PageShift
|
|
+ default:
|
|
+ return false
|
|
+ }
|
|
+}
|
|
+
|
|
// TODO: spanOf and spanOfUnchecked are open-coded in a lot of places.
|
|
// Use the functions instead.
|
|
|