diff --git a/src/runtime/gc_blocks.go b/src/runtime/gc_blocks.go index 01031ac89c..c995f0f14f 100644 --- a/src/runtime/gc_blocks.go +++ b/src/runtime/gc_blocks.go @@ -32,6 +32,7 @@ package runtime import ( "internal/task" + "math/bits" "runtime/interrupt" "unsafe" ) @@ -74,6 +75,11 @@ const ( blockStateMask blockState = 3 // 11 ) +// blockStateByteAllTails is a byte containing 4 times blockStateTail bits. +// It assumes there are 2 state bits per block, otherwise it might have to be +// turned into variable and assigned using inline function. +const blockStateByteAllTails = uint8(blockStateTail<<(stateBits*3) | blockStateTail<<(stateBits*2) | blockStateTail<<(stateBits*1) | blockStateTail<<(stateBits*0)) + // String returns a human-readable version of the block state, for debugging. func (s blockState) String() string { switch s { @@ -121,9 +127,32 @@ func (b gcBlock) address() uintptr { // points to an allocated object. It returns the same block if this block // already points to the head. func (b gcBlock) findHead() gcBlock { - for b.state() == blockStateTail { - b-- + stateBytePtr := (*uint8)(unsafe.Add(metadataStart, b/blocksPerStateByte)) + + // XOR the stateByte with byte containing all tails to turn tail bits to 0 + // and shift out the bits that are not part of the object + stateByte := ((*stateBytePtr) ^ blockStateByteAllTails) << ((blocksPerStateByte - (b%blocksPerStateByte + 1)) * stateBits) + // if stateByte is 0 that means all blocks are tails so we loop trough subsequent states, + // byte at a time to find the first byte that is not all tails + if stateByte == 0 { + // subtract the number of object blocks that were in the first byte + b -= (b%blocksPerStateByte + 1) + // skip to next byte + stateBytePtr = (*uint8)(unsafe.Add(unsafe.Pointer(stateBytePtr), -1)) + // loop until state byte is not all tails + for (*stateBytePtr)^blockStateByteAllTails == 0 { + stateBytePtr = (*uint8)(unsafe.Add(unsafe.Pointer(stateBytePtr), -1)) + b -= blocksPerStateByte + } + // set stateByte variable to the first byte that is not all tails and turn all tail bits to zeroes + stateByte = (*stateBytePtr) ^ blockStateByteAllTails } + + // at this point stateByte is set to the first state byte of the object that we encountered which is not all tails + // and all tail bits in it are turned to zero. We count number of bytes that are 0 (tail) using LeadingZeros8 + // and divide it by stateBits to get the number of tail blocks in state bits. + b -= gcBlock(bits.LeadingZeros8(stateByte) / stateBits) + if gcAsserts { if b.state() != blockStateHead && b.state() != blockStateMark { runtimePanic("gc: found tail without head")