mirror of
https://github.com/apple/swift-collections
synced 2026-11-22 04:20:40 +00:00
[TrailingArray] Handle trailing arrays where the element has greater alignment than the header
When this happens, we over-allocate and use the alignment of the elements, and then move the start of the header after the point of allocation such that the properly-aligned elements follow it immediately.
This commit is contained in:
1 parent
aa36f635ad
commit
43367db377
2 files changed
+247
-88
No files matched your search
@@ -10,9 +10,17 @@
|
||||
public struct TrailingArray<Header: TrailingElements>: ~Copyable
|
||||
where Header: ~Copyable
|
||||
{
|
||||
/// The underlying storage.
|
||||
/// A pointer to the raw, underlying storage. This may point before the
|
||||
/// Header instance in cases where the alignment of Element exceeds that
|
||||
/// of the Header and we have overallocated to compensate.
|
||||
@usableFromInline
|
||||
var _pointer: UnsafeMutablePointer<Header>
|
||||
var _storage: UnsafeMutableRawPointer
|
||||
|
||||
/// Pointer to the header.
|
||||
@_alwaysEmitIntoClient
|
||||
var _pointer: UnsafeMutablePointer<Header> {
|
||||
Self.headerPointer(fromStorage: _storage)
|
||||
}
|
||||
|
||||
/// The element type stored within the buffer.
|
||||
public typealias Element = Header.Element
|
||||
@@ -22,12 +30,11 @@ public struct TrailingArray<Header: TrailingElements>: ~Copyable
|
||||
/// memory safety.
|
||||
@_alwaysEmitIntoClient
|
||||
private init(_headerOnly header: consuming Header) {
|
||||
_pointer = UnsafeMutableRawPointer.allocate(
|
||||
byteCount: Self.allocationSize(
|
||||
header: header
|
||||
),
|
||||
alignment: MemoryLayout<Header>.alignment
|
||||
).assumingMemoryBound(to: Header.self)
|
||||
let (bytes, alignment) = Self.allocationSize(header: header)
|
||||
_storage = UnsafeMutableRawPointer.allocate(
|
||||
byteCount: bytes,
|
||||
alignment: alignment
|
||||
)
|
||||
|
||||
_pointer.initialize(to: header)
|
||||
}
|
||||
@@ -69,7 +76,7 @@ public struct TrailingArray<Header: TrailingElements>: ~Copyable
|
||||
deinit {
|
||||
rawElements.deinitialize()
|
||||
_pointer.deinitialize(count: 1)
|
||||
_pointer.deallocate()
|
||||
_storage.deallocate()
|
||||
}
|
||||
|
||||
/// Take ownership over a pointer to memory containing the header followed
|
||||
@@ -77,19 +84,39 @@ public struct TrailingArray<Header: TrailingElements>: ~Copyable
|
||||
///
|
||||
/// Once this managed buffer instance is no longer used, this memory will
|
||||
/// be freed.
|
||||
/// - Parameters:
|
||||
/// - pointer: A pointer to the header, which should already have been
|
||||
/// initialized by the caller.
|
||||
/// - storage: A pointer to the storage containing the header and
|
||||
/// pointers. This will usually be `UnsafeMutableRawPointer(pointer)`,
|
||||
/// but may precede `pointer` if the elements have greater alignment
|
||||
/// than the header.
|
||||
@_alwaysEmitIntoClient
|
||||
public init(consuming pointer: UnsafeMutablePointer<Header>) {
|
||||
self._pointer = pointer
|
||||
public init(
|
||||
consuming pointer: UnsafeMutablePointer<Header>,
|
||||
storage: UnsafeMutableRawPointer
|
||||
) {
|
||||
self._storage = storage
|
||||
precondition(
|
||||
self._pointer == pointer,
|
||||
"header pointer does not account for the alignment of the elements"
|
||||
)
|
||||
}
|
||||
|
||||
/// Return the pointer to the underlying memory, including ownership over
|
||||
/// that memory. The underlying storage will not be freed by this buffer;
|
||||
/// it is the responsibility of the caller.
|
||||
/// it is the responsibility of the caller. The header pointer and
|
||||
/// underlying storage are returned separately, to handle cases where
|
||||
/// the alignment of the elements is greater than than of the header.
|
||||
@_alwaysEmitIntoClient
|
||||
public consuming func leakStorage() -> UnsafeMutablePointer<Header> {
|
||||
public consuming func leakStorage() -> (
|
||||
pointer: UnsafeMutablePointer<Header>,
|
||||
storage: UnsafeMutableRawPointer
|
||||
) {
|
||||
let pointer = self._pointer
|
||||
let storage = _storage
|
||||
discard self
|
||||
return pointer
|
||||
return (pointer, storage)
|
||||
}
|
||||
|
||||
/// Access the header portion of the buffer.
|
||||
@@ -189,10 +216,53 @@ public struct TrailingArray<Header: TrailingElements>: ~Copyable
|
||||
return try body(_pointer, rawElements)
|
||||
}
|
||||
|
||||
/// Determine the allocation size needed for the given header value.
|
||||
/// Determine the allocation size and alignment needed for the given header
|
||||
/// value along with its trailing elements.
|
||||
///
|
||||
/// In cases where the element's alignment exceeds that of the header,
|
||||
/// this operation will suggest overallocation so that the elements can be
|
||||
/// properly aligned following the header.
|
||||
@_alwaysEmitIntoClient
|
||||
public static func allocationSize(header: borrowing Header) -> Int {
|
||||
MemoryLayout<Header>.size + MemoryLayout<Element>.stride * header.trailingCount
|
||||
static func allocationSize(header: borrowing Header) -> (size: Int, alignment: Int) {
|
||||
// The number of bytes needed to contain the header and elements,
|
||||
// assuming that there are no alignment issues.
|
||||
let numBytes = MemoryLayout<Header>.stride + MemoryLayout<Element>.stride * header.trailingCount
|
||||
|
||||
let headerAlignment = MemoryLayout<Header>.alignment
|
||||
let elementAlignment = MemoryLayout<Element>.alignment
|
||||
|
||||
// If the header provides sufficient alignment for the elements,
|
||||
// we're done.
|
||||
if elementAlignment <= headerAlignment {
|
||||
return (numBytes, MemoryLayout<Header>.alignment)
|
||||
}
|
||||
|
||||
// We may have to slide an allocation by up to the difference between
|
||||
// the element and header alignments to ensure that the elements are
|
||||
// appropriately aligned.
|
||||
return (numBytes + elementAlignment - headerAlignment,
|
||||
MemoryLayout<Element>.alignment)
|
||||
}
|
||||
|
||||
/// Given storage that is large enough to accommodate padding + the header
|
||||
/// + the elements, return the header pointer from the storage pointer.
|
||||
@_alwaysEmitIntoClient
|
||||
static func headerPointer(fromStorage storage: UnsafeMutableRawPointer) -> UnsafeMutablePointer<Header> {
|
||||
let headerAlignment = MemoryLayout<Header>.alignment
|
||||
let elementAlignment = MemoryLayout<Element>.alignment
|
||||
|
||||
// Normal case: the header has sufficient alignment to include the
|
||||
// elements, so the storage refers directly to the header.
|
||||
if elementAlignment <= headerAlignment {
|
||||
return storage.assumingMemoryBound(to: Header.self)
|
||||
}
|
||||
|
||||
// The header is stored right before the elements, with padding bytes
|
||||
// between the start of the allocation and up to the header.
|
||||
return storage.advanced(by: MemoryLayout<Header>.stride)
|
||||
.alignedUp(for: Element.self)
|
||||
.advanced(by: -MemoryLayout<Header>.stride)
|
||||
.assumingMemoryBound(to: Header.self)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -218,29 +288,9 @@ extension TrailingArray where Header: Copyable {
|
||||
repeating element: Element,
|
||||
body: (inout TrailingArray<Header>) throws(E) -> R
|
||||
) throws(E) -> R {
|
||||
return try TrailingPadding<Header>.withTemporaryValue(
|
||||
header: header,
|
||||
totalSize: allocationSize(header: header)
|
||||
) { (storage) throws(E) in
|
||||
/// Create a managed buffer over that temporary storage.
|
||||
var managedBuffer = TrailingArray(consuming: storage._pointer)
|
||||
managedBuffer.rawElements.initialize(repeating: element)
|
||||
|
||||
do throws(E) {
|
||||
let result = try body(&managedBuffer)
|
||||
|
||||
// Tell the managed buffer not to free the storage.
|
||||
let finalPointer = managedBuffer.leakStorage()
|
||||
precondition(finalPointer == storage._pointer)
|
||||
|
||||
return result
|
||||
} catch {
|
||||
// Tell the managed buffer not to free the storage.
|
||||
let finalPointer = managedBuffer.leakStorage()
|
||||
precondition(finalPointer == storage._pointer)
|
||||
|
||||
throw error
|
||||
}
|
||||
return try _withTemporaryValue(header: header, uninitializedTrailingElements: ()) { (value) throws(E) in
|
||||
value.rawElements.initialize(repeating: element)
|
||||
return try body(&value)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -256,33 +306,64 @@ extension TrailingArray where Header: Copyable {
|
||||
initializingTrailingElementsWith initializer: (inout OutputSpan<Element>) throws(E) -> Void,
|
||||
body: (inout TrailingArray<Header>) throws(E) -> R
|
||||
) throws(E) -> R {
|
||||
return try TrailingPadding<Header>.withTemporaryValue(
|
||||
header: header,
|
||||
totalSize: allocationSize(header: header)
|
||||
) { (storage) throws(E) in
|
||||
/// Create a managed buffer over that temporary storage.
|
||||
var managedBuffer = TrailingArray(consuming: storage._pointer)
|
||||
try managedBuffer._initializeTrailingElements(initializer: initializer)
|
||||
|
||||
do throws(E) {
|
||||
let result = try body(&managedBuffer)
|
||||
|
||||
// Tell the managed buffer not to free the storage.
|
||||
let finalPointer = managedBuffer.leakStorage()
|
||||
precondition(finalPointer == storage._pointer)
|
||||
|
||||
return result
|
||||
} catch {
|
||||
// Tell the managed buffer not to free the storage.
|
||||
let finalPointer = managedBuffer.leakStorage()
|
||||
precondition(finalPointer == storage._pointer)
|
||||
|
||||
throw error
|
||||
}
|
||||
return try _withTemporaryValue(header: header, uninitializedTrailingElements: ()) { (value) throws(E) in
|
||||
try value._initializeTrailingElements(initializer: initializer)
|
||||
return try body(&value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension TrailingArray {
|
||||
/// Create a temporary intrusive managed buffer for the given header, whose
|
||||
/// trailing elements are left uninitialized. That instance is provided to
|
||||
/// the given `body` to operate on for the duration of the
|
||||
/// call. The temporary is allocated on the stack, unless it is very
|
||||
/// large according to `withUnsafeTemporaryAllocation`.
|
||||
@_alwaysEmitIntoClient
|
||||
@unsafe
|
||||
static func _withTemporaryValue<R: ~Copyable, E>(
|
||||
header: Header,
|
||||
uninitializedTrailingElements: (),
|
||||
body: (inout TrailingArray<Header>) throws(E) -> R
|
||||
) throws(E) -> R {
|
||||
let (numBytes, alignment) = allocationSize(header: header)
|
||||
|
||||
// Allocate temporary storage large enough for the value we need.
|
||||
let result: Result<R, E> = withUnsafeTemporaryAllocation(
|
||||
byteCount: numBytes,
|
||||
alignment: alignment
|
||||
) { buffer in
|
||||
// Initialize the header within the temporary allocation.
|
||||
let storagePointer = buffer.baseAddress!
|
||||
let headerPointer = TrailingArray.headerPointer(fromStorage: storagePointer)
|
||||
headerPointer.initialize(to: header)
|
||||
|
||||
/// Create a trailing array over that temporary storage.
|
||||
var managedBuffer = TrailingArray(consuming: headerPointer,
|
||||
storage: storagePointer)
|
||||
let resultOrError: Result<R, E>
|
||||
do throws(E) {
|
||||
resultOrError = .success(try body(&managedBuffer))
|
||||
} catch {
|
||||
resultOrError = .failure(error)
|
||||
}
|
||||
|
||||
// Deinitialize the elements and header.
|
||||
managedBuffer.rawElements.deinitialize()
|
||||
managedBuffer._pointer.deinitialize(count: 1)
|
||||
|
||||
// Tell the trailing buffer not to free the storage.
|
||||
let (finalPointer, finalStorage) = managedBuffer.leakStorage()
|
||||
precondition(finalPointer == headerPointer)
|
||||
precondition(finalStorage == storagePointer)
|
||||
|
||||
return resultOrError
|
||||
}
|
||||
|
||||
return try result.get()
|
||||
}
|
||||
}
|
||||
|
||||
extension TrailingArray where Header.Element: BitwiseCopyable {
|
||||
/// Create a temporary intrusive managed buffer for the given header, whose
|
||||
/// trailing elements are left uninitialized. That instance is provided to
|
||||
@@ -296,28 +377,8 @@ extension TrailingArray where Header.Element: BitwiseCopyable {
|
||||
uninitializedTrailingElements: (),
|
||||
body: (inout TrailingArray<Header>) throws(E) -> R
|
||||
) throws(E) -> R {
|
||||
return try TrailingPadding<Header>.withTemporaryValue(
|
||||
header: header,
|
||||
totalSize: allocationSize(header: header)
|
||||
) { (storage) throws(E) in
|
||||
/// Create a managed buffer over that temporary storage.
|
||||
var managedBuffer = TrailingArray(consuming: storage._pointer)
|
||||
|
||||
do throws(E) {
|
||||
let result = try body(&managedBuffer)
|
||||
|
||||
// Tell the managed buffer not to free the storage.
|
||||
let finalPointer = managedBuffer.leakStorage()
|
||||
precondition(finalPointer == storage._pointer)
|
||||
|
||||
return result
|
||||
} catch {
|
||||
// Tell the managed buffer not to free the storage.
|
||||
let finalPointer = managedBuffer.leakStorage()
|
||||
precondition(finalPointer == storage._pointer)
|
||||
|
||||
throw error
|
||||
}
|
||||
return try _withTemporaryValue(header: header, uninitializedTrailingElements: ()) { (value) throws(E) in
|
||||
try body(&value)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,9 +15,31 @@ extension Coordinates: TrailingElements {
|
||||
var trailingCount: Int { numCoordinates }
|
||||
}
|
||||
|
||||
struct OneByteWithPointers: TrailingElements {
|
||||
var byte: UInt8
|
||||
|
||||
init(count: Int) {
|
||||
self.byte = UInt8(count)
|
||||
}
|
||||
|
||||
typealias Element = OpaquePointer
|
||||
var trailingCount: Int { Int(byte) }
|
||||
}
|
||||
|
||||
struct ManyBytesWithPointers: TrailingElements {
|
||||
var bytes: (UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8)
|
||||
|
||||
init(count: Int) {
|
||||
self.bytes = (UInt8(count), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
|
||||
}
|
||||
|
||||
typealias Element = OpaquePointer
|
||||
var trailingCount: Int { Int(bytes.0) }
|
||||
}
|
||||
|
||||
@Suite("Trailing elements tests")
|
||||
struct TrailingElementsTests {
|
||||
@Test func simpleCoordinates() async throws {
|
||||
@Test func simpleCoordinates() {
|
||||
var coords = TrailingArray(header: Coordinates(numCoordinates: 3)) { outputSpan in
|
||||
outputSpan.append(Point(x: 1, y: 2))
|
||||
outputSpan.append(Point(x: 2, y: 3))
|
||||
@@ -32,7 +54,7 @@ struct TrailingElementsTests {
|
||||
#expect(coords[1].y == 17)
|
||||
}
|
||||
|
||||
@Test func repeatingInit() async throws {
|
||||
@Test func repeatingInit() {
|
||||
var coords = TrailingArray(header: Coordinates(numCoordinates: 3), repeating: Point(x: 1, y: 1))
|
||||
|
||||
#expect(coords[0].x == 1)
|
||||
@@ -45,7 +67,7 @@ struct TrailingElementsTests {
|
||||
#expect(coords[2].x == 1)
|
||||
}
|
||||
|
||||
@Test func temporaryCoordinates() async throws {
|
||||
@Test func temporaryCoordinates() {
|
||||
TrailingArray.withTemporaryValue(header: Coordinates(numCoordinates: 3)) { outputSpan in
|
||||
outputSpan.append(Point(x: 1, y: 2))
|
||||
outputSpan.append(Point(x: 2, y: 3))
|
||||
@@ -56,4 +78,80 @@ struct TrailingElementsTests {
|
||||
#expect(coords[2].x == 3)
|
||||
}
|
||||
}
|
||||
|
||||
@Test(arguments: 0 ... 10)
|
||||
func underalignedByteOnHeap(count: Int) {
|
||||
var pointers = TrailingArray(header: OneByteWithPointers(count: count)) { outputSpan in
|
||||
for i in 0..<count {
|
||||
outputSpan.append(OpaquePointer(bitPattern: i+1)!)
|
||||
}
|
||||
}
|
||||
|
||||
for i in 0..<count {
|
||||
#expect(pointers[i] == OpaquePointer(bitPattern: i+1))
|
||||
}
|
||||
|
||||
pointers.withUnsafeMutablePointers { headerPtr, elementsPtr in
|
||||
#expect(
|
||||
UnsafeMutableRawPointer(headerPtr.advanced(by: MemoryLayout<OneByteWithPointers>.stride)) ==
|
||||
UnsafeMutableRawPointer(elementsPtr.baseAddress))
|
||||
}
|
||||
}
|
||||
|
||||
@Test(arguments: 0 ... 10)
|
||||
func underalignedByteOnStack(count: Int) {
|
||||
TrailingArray.withTemporaryValue(header: OneByteWithPointers(count: count)) { outputSpan in
|
||||
for i in 0..<count {
|
||||
outputSpan.append(OpaquePointer(bitPattern: i+1)!)
|
||||
}
|
||||
} body: { pointers in
|
||||
for i in 0..<count {
|
||||
#expect(pointers[i] == OpaquePointer(bitPattern: i+1))
|
||||
}
|
||||
|
||||
pointers.withUnsafeMutablePointers { headerPtr, elementsPtr in
|
||||
#expect(
|
||||
UnsafeMutableRawPointer(headerPtr.advanced(by: MemoryLayout<OneByteWithPointers>.stride)) ==
|
||||
UnsafeMutableRawPointer(elementsPtr.baseAddress))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test(arguments: 0 ... 10)
|
||||
func underalignedBytesOnHeap(count: Int) {
|
||||
var pointers = TrailingArray(header: ManyBytesWithPointers(count: count)) { outputSpan in
|
||||
for i in 0..<count {
|
||||
outputSpan.append(OpaquePointer(bitPattern: i+1)!)
|
||||
}
|
||||
}
|
||||
|
||||
for i in 0..<count {
|
||||
#expect(pointers[i] == OpaquePointer(bitPattern: i+1))
|
||||
}
|
||||
|
||||
pointers.withUnsafeMutablePointers { headerPtr, elementsPtr in
|
||||
#expect(
|
||||
UnsafeMutableRawPointer(headerPtr.advanced(by: MemoryLayout<OneByteWithPointers>.stride)) ==
|
||||
UnsafeMutableRawPointer(elementsPtr.baseAddress))
|
||||
}
|
||||
}
|
||||
|
||||
@Test(arguments: 0 ... 10)
|
||||
func underalignedBytesOnStack(count: Int) {
|
||||
TrailingArray.withTemporaryValue(header: ManyBytesWithPointers(count: count)) { outputSpan in
|
||||
for i in 0..<count {
|
||||
outputSpan.append(OpaquePointer(bitPattern: i+1)!)
|
||||
}
|
||||
} body: { pointers in
|
||||
for i in 0..<count {
|
||||
#expect(pointers[i] == OpaquePointer(bitPattern: i+1))
|
||||
}
|
||||
|
||||
pointers.withUnsafeMutablePointers { headerPtr, elementsPtr in
|
||||
#expect(
|
||||
UnsafeMutableRawPointer(headerPtr.advanced(by: MemoryLayout<OneByteWithPointers>.stride)) ==
|
||||
UnsafeMutableRawPointer(elementsPtr.baseAddress))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user