[TrailingArray] Handle trailing arrays where the element has greater alignment than the header

When this happens, we over-allocate and use the alignment of the
elements, and then move the start of the header after the point of
allocation such that the properly-aligned elements follow it
immediately.
This commit is contained in:
Doug Gregor authored and Guillaume Lessard committed 2025-09-12 11:23:07 -07:00
1 parent aa36f635ad
commit 43367db377
2 files changed
+247 -88

No files matched your search

@@ -10,9 +10,17 @@
public struct TrailingArray<Header: TrailingElements>: ~Copyable
where Header: ~Copyable
{
/// The underlying storage.
/// A pointer to the raw, underlying storage. This may point before the
/// Header instance in cases where the alignment of Element exceeds that
/// of the Header and we have overallocated to compensate.
@usableFromInline
var _pointer: UnsafeMutablePointer<Header>
var _storage: UnsafeMutableRawPointer
/// Pointer to the header.
@_alwaysEmitIntoClient
var _pointer: UnsafeMutablePointer<Header> {
Self.headerPointer(fromStorage: _storage)
}
/// The element type stored within the buffer.
public typealias Element = Header.Element
@@ -22,12 +30,11 @@ public struct TrailingArray<Header: TrailingElements>: ~Copyable
/// memory safety.
@_alwaysEmitIntoClient
private init(_headerOnly header: consuming Header) {
_pointer = UnsafeMutableRawPointer.allocate(
byteCount: Self.allocationSize(
header: header
),
alignment: MemoryLayout<Header>.alignment
).assumingMemoryBound(to: Header.self)
let (bytes, alignment) = Self.allocationSize(header: header)
_storage = UnsafeMutableRawPointer.allocate(
byteCount: bytes,
alignment: alignment
)
_pointer.initialize(to: header)
}
@@ -69,7 +76,7 @@ public struct TrailingArray<Header: TrailingElements>: ~Copyable
deinit {
rawElements.deinitialize()
_pointer.deinitialize(count: 1)
_pointer.deallocate()
_storage.deallocate()
}
/// Take ownership over a pointer to memory containing the header followed
@@ -77,19 +84,39 @@ public struct TrailingArray<Header: TrailingElements>: ~Copyable
///
/// Once this managed buffer instance is no longer used, this memory will
/// be freed.
/// - Parameters:
/// - pointer: A pointer to the header, which should already have been
/// initialized by the caller.
/// - storage: A pointer to the storage containing the header and
/// pointers. This will usually be `UnsafeMutableRawPointer(pointer)`,
/// but may precede `pointer` if the elements have greater alignment
/// than the header.
@_alwaysEmitIntoClient
public init(consuming pointer: UnsafeMutablePointer<Header>) {
self._pointer = pointer
public init(
consuming pointer: UnsafeMutablePointer<Header>,
storage: UnsafeMutableRawPointer
) {
self._storage = storage
precondition(
self._pointer == pointer,
"header pointer does not account for the alignment of the elements"
)
}
/// Return the pointer to the underlying memory, including ownership over
/// that memory. The underlying storage will not be freed by this buffer;
/// it is the responsibility of the caller.
/// it is the responsibility of the caller. The header pointer and
/// underlying storage are returned separately, to handle cases where
/// the alignment of the elements is greater than than of the header.
@_alwaysEmitIntoClient
public consuming func leakStorage() -> UnsafeMutablePointer<Header> {
public consuming func leakStorage() -> (
pointer: UnsafeMutablePointer<Header>,
storage: UnsafeMutableRawPointer
) {
let pointer = self._pointer
let storage = _storage
discard self
return pointer
return (pointer, storage)
}
/// Access the header portion of the buffer.
@@ -189,10 +216,53 @@ public struct TrailingArray<Header: TrailingElements>: ~Copyable
return try body(_pointer, rawElements)
}
/// Determine the allocation size needed for the given header value.
/// Determine the allocation size and alignment needed for the given header
/// value along with its trailing elements.
///
/// In cases where the element's alignment exceeds that of the header,
/// this operation will suggest overallocation so that the elements can be
/// properly aligned following the header.
@_alwaysEmitIntoClient
public static func allocationSize(header: borrowing Header) -> Int {
MemoryLayout<Header>.size + MemoryLayout<Element>.stride * header.trailingCount
static func allocationSize(header: borrowing Header) -> (size: Int, alignment: Int) {
// The number of bytes needed to contain the header and elements,
// assuming that there are no alignment issues.
let numBytes = MemoryLayout<Header>.stride + MemoryLayout<Element>.stride * header.trailingCount
let headerAlignment = MemoryLayout<Header>.alignment
let elementAlignment = MemoryLayout<Element>.alignment
// If the header provides sufficient alignment for the elements,
// we're done.
if elementAlignment <= headerAlignment {
return (numBytes, MemoryLayout<Header>.alignment)
}
// We may have to slide an allocation by up to the difference between
// the element and header alignments to ensure that the elements are
// appropriately aligned.
return (numBytes + elementAlignment - headerAlignment,
MemoryLayout<Element>.alignment)
}
/// Given storage that is large enough to accommodate padding + the header
/// + the elements, return the header pointer from the storage pointer.
@_alwaysEmitIntoClient
static func headerPointer(fromStorage storage: UnsafeMutableRawPointer) -> UnsafeMutablePointer<Header> {
let headerAlignment = MemoryLayout<Header>.alignment
let elementAlignment = MemoryLayout<Element>.alignment
// Normal case: the header has sufficient alignment to include the
// elements, so the storage refers directly to the header.
if elementAlignment <= headerAlignment {
return storage.assumingMemoryBound(to: Header.self)
}
// The header is stored right before the elements, with padding bytes
// between the start of the allocation and up to the header.
return storage.advanced(by: MemoryLayout<Header>.stride)
.alignedUp(for: Element.self)
.advanced(by: -MemoryLayout<Header>.stride)
.assumingMemoryBound(to: Header.self)
}
}
@@ -218,29 +288,9 @@ extension TrailingArray where Header: Copyable {
repeating element: Element,
body: (inout TrailingArray<Header>) throws(E) -> R
) throws(E) -> R {
return try TrailingPadding<Header>.withTemporaryValue(
header: header,
totalSize: allocationSize(header: header)
) { (storage) throws(E) in
/// Create a managed buffer over that temporary storage.
var managedBuffer = TrailingArray(consuming: storage._pointer)
managedBuffer.rawElements.initialize(repeating: element)
do throws(E) {
let result = try body(&managedBuffer)
// Tell the managed buffer not to free the storage.
let finalPointer = managedBuffer.leakStorage()
precondition(finalPointer == storage._pointer)
return result
} catch {
// Tell the managed buffer not to free the storage.
let finalPointer = managedBuffer.leakStorage()
precondition(finalPointer == storage._pointer)
throw error
}
return try _withTemporaryValue(header: header, uninitializedTrailingElements: ()) { (value) throws(E) in
value.rawElements.initialize(repeating: element)
return try body(&value)
}
}
@@ -256,33 +306,64 @@ extension TrailingArray where Header: Copyable {
initializingTrailingElementsWith initializer: (inout OutputSpan<Element>) throws(E) -> Void,
body: (inout TrailingArray<Header>) throws(E) -> R
) throws(E) -> R {
return try TrailingPadding<Header>.withTemporaryValue(
header: header,
totalSize: allocationSize(header: header)
) { (storage) throws(E) in
/// Create a managed buffer over that temporary storage.
var managedBuffer = TrailingArray(consuming: storage._pointer)
try managedBuffer._initializeTrailingElements(initializer: initializer)
do throws(E) {
let result = try body(&managedBuffer)
// Tell the managed buffer not to free the storage.
let finalPointer = managedBuffer.leakStorage()
precondition(finalPointer == storage._pointer)
return result
} catch {
// Tell the managed buffer not to free the storage.
let finalPointer = managedBuffer.leakStorage()
precondition(finalPointer == storage._pointer)
throw error
}
return try _withTemporaryValue(header: header, uninitializedTrailingElements: ()) { (value) throws(E) in
try value._initializeTrailingElements(initializer: initializer)
return try body(&value)
}
}
}
extension TrailingArray {
/// Create a temporary intrusive managed buffer for the given header, whose
/// trailing elements are left uninitialized. That instance is provided to
/// the given `body` to operate on for the duration of the
/// call. The temporary is allocated on the stack, unless it is very
/// large according to `withUnsafeTemporaryAllocation`.
@_alwaysEmitIntoClient
@unsafe
static func _withTemporaryValue<R: ~Copyable, E>(
header: Header,
uninitializedTrailingElements: (),
body: (inout TrailingArray<Header>) throws(E) -> R
) throws(E) -> R {
let (numBytes, alignment) = allocationSize(header: header)
// Allocate temporary storage large enough for the value we need.
let result: Result<R, E> = withUnsafeTemporaryAllocation(
byteCount: numBytes,
alignment: alignment
) { buffer in
// Initialize the header within the temporary allocation.
let storagePointer = buffer.baseAddress!
let headerPointer = TrailingArray.headerPointer(fromStorage: storagePointer)
headerPointer.initialize(to: header)
/// Create a trailing array over that temporary storage.
var managedBuffer = TrailingArray(consuming: headerPointer,
storage: storagePointer)
let resultOrError: Result<R, E>
do throws(E) {
resultOrError = .success(try body(&managedBuffer))
} catch {
resultOrError = .failure(error)
}
// Deinitialize the elements and header.
managedBuffer.rawElements.deinitialize()
managedBuffer._pointer.deinitialize(count: 1)
// Tell the trailing buffer not to free the storage.
let (finalPointer, finalStorage) = managedBuffer.leakStorage()
precondition(finalPointer == headerPointer)
precondition(finalStorage == storagePointer)
return resultOrError
}
return try result.get()
}
}
extension TrailingArray where Header.Element: BitwiseCopyable {
/// Create a temporary intrusive managed buffer for the given header, whose
/// trailing elements are left uninitialized. That instance is provided to
@@ -296,28 +377,8 @@ extension TrailingArray where Header.Element: BitwiseCopyable {
uninitializedTrailingElements: (),
body: (inout TrailingArray<Header>) throws(E) -> R
) throws(E) -> R {
return try TrailingPadding<Header>.withTemporaryValue(
header: header,
totalSize: allocationSize(header: header)
) { (storage) throws(E) in
/// Create a managed buffer over that temporary storage.
var managedBuffer = TrailingArray(consuming: storage._pointer)
do throws(E) {
let result = try body(&managedBuffer)
// Tell the managed buffer not to free the storage.
let finalPointer = managedBuffer.leakStorage()
precondition(finalPointer == storage._pointer)
return result
} catch {
// Tell the managed buffer not to free the storage.
let finalPointer = managedBuffer.leakStorage()
precondition(finalPointer == storage._pointer)
throw error
}
return try _withTemporaryValue(header: header, uninitializedTrailingElements: ()) { (value) throws(E) in
try body(&value)
}
}
}
@@ -15,9 +15,31 @@ extension Coordinates: TrailingElements {
var trailingCount: Int { numCoordinates }
}
struct OneByteWithPointers: TrailingElements {
var byte: UInt8
init(count: Int) {
self.byte = UInt8(count)
}
typealias Element = OpaquePointer
var trailingCount: Int { Int(byte) }
}
struct ManyBytesWithPointers: TrailingElements {
var bytes: (UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8)
init(count: Int) {
self.bytes = (UInt8(count), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
}
typealias Element = OpaquePointer
var trailingCount: Int { Int(bytes.0) }
}
@Suite("Trailing elements tests")
struct TrailingElementsTests {
@Test func simpleCoordinates() async throws {
@Test func simpleCoordinates() {
var coords = TrailingArray(header: Coordinates(numCoordinates: 3)) { outputSpan in
outputSpan.append(Point(x: 1, y: 2))
outputSpan.append(Point(x: 2, y: 3))
@@ -32,7 +54,7 @@ struct TrailingElementsTests {
#expect(coords[1].y == 17)
}
@Test func repeatingInit() async throws {
@Test func repeatingInit() {
var coords = TrailingArray(header: Coordinates(numCoordinates: 3), repeating: Point(x: 1, y: 1))
#expect(coords[0].x == 1)
@@ -45,7 +67,7 @@ struct TrailingElementsTests {
#expect(coords[2].x == 1)
}
@Test func temporaryCoordinates() async throws {
@Test func temporaryCoordinates() {
TrailingArray.withTemporaryValue(header: Coordinates(numCoordinates: 3)) { outputSpan in
outputSpan.append(Point(x: 1, y: 2))
outputSpan.append(Point(x: 2, y: 3))
@@ -56,4 +78,80 @@ struct TrailingElementsTests {
#expect(coords[2].x == 3)
}
}
@Test(arguments: 0 ... 10)
func underalignedByteOnHeap(count: Int) {
var pointers = TrailingArray(header: OneByteWithPointers(count: count)) { outputSpan in
for i in 0..<count {
outputSpan.append(OpaquePointer(bitPattern: i+1)!)
}
}
for i in 0..<count {
#expect(pointers[i] == OpaquePointer(bitPattern: i+1))
}
pointers.withUnsafeMutablePointers { headerPtr, elementsPtr in
#expect(
UnsafeMutableRawPointer(headerPtr.advanced(by: MemoryLayout<OneByteWithPointers>.stride)) ==
UnsafeMutableRawPointer(elementsPtr.baseAddress))
}
}
@Test(arguments: 0 ... 10)
func underalignedByteOnStack(count: Int) {
TrailingArray.withTemporaryValue(header: OneByteWithPointers(count: count)) { outputSpan in
for i in 0..<count {
outputSpan.append(OpaquePointer(bitPattern: i+1)!)
}
} body: { pointers in
for i in 0..<count {
#expect(pointers[i] == OpaquePointer(bitPattern: i+1))
}
pointers.withUnsafeMutablePointers { headerPtr, elementsPtr in
#expect(
UnsafeMutableRawPointer(headerPtr.advanced(by: MemoryLayout<OneByteWithPointers>.stride)) ==
UnsafeMutableRawPointer(elementsPtr.baseAddress))
}
}
}
@Test(arguments: 0 ... 10)
func underalignedBytesOnHeap(count: Int) {
var pointers = TrailingArray(header: ManyBytesWithPointers(count: count)) { outputSpan in
for i in 0..<count {
outputSpan.append(OpaquePointer(bitPattern: i+1)!)
}
}
for i in 0..<count {
#expect(pointers[i] == OpaquePointer(bitPattern: i+1))
}
pointers.withUnsafeMutablePointers { headerPtr, elementsPtr in
#expect(
UnsafeMutableRawPointer(headerPtr.advanced(by: MemoryLayout<OneByteWithPointers>.stride)) ==
UnsafeMutableRawPointer(elementsPtr.baseAddress))
}
}
@Test(arguments: 0 ... 10)
func underalignedBytesOnStack(count: Int) {
TrailingArray.withTemporaryValue(header: ManyBytesWithPointers(count: count)) { outputSpan in
for i in 0..<count {
outputSpan.append(OpaquePointer(bitPattern: i+1)!)
}
} body: { pointers in
for i in 0..<count {
#expect(pointers[i] == OpaquePointer(bitPattern: i+1))
}
pointers.withUnsafeMutablePointers { headerPtr, elementsPtr in
#expect(
UnsafeMutableRawPointer(headerPtr.advanced(by: MemoryLayout<OneByteWithPointers>.stride)) ==
UnsafeMutableRawPointer(elementsPtr.baseAddress))
}
}
}
}