mirror of
https://github.com/apple/swift-nio-extras
synced 2026-11-22 04:30:42 +00:00
Keep zlib streams at stable addresses (#321)
### Motivation: zlib stores the address of `z_stream` in its internal state and expects that address to remain stable across calls. Storing `z_stream` inline in movable Swift value types does not provide that guarantee and can cause `Z_STREAM_ERROR` during cleanup. Fixes #308. ### Modifications: - Make the internal compressor and decompressor reference owners of separately allocated `z_stream` storage. - Pass the stable pointer to every zlib initialization, processing, and cleanup call. - Keep lifecycle state with the pointer owner and make conditional cleanup idempotent. - Add regression tests covering failed initialization recovery, multiple compression and decompression calls, repeated lifecycles, and deallocation cleanup. ### Result: zlib observes the same `z_stream` address for each complete lifecycle, while stream cleanup remains single-owner and safe against repeated shutdown.
This commit is contained in:
1 parent
aedcd7e2d4
commit
4ba1f167a7
6 files changed
+194
-59
No files matched your search
@@ -53,21 +53,37 @@ public enum NIOCompression: Sendable {
|
||||
}
|
||||
|
||||
/// Data compression utility.
|
||||
struct Compressor {
|
||||
private var stream = cnioextras_z_stream()
|
||||
final class Compressor {
|
||||
// zlib stores the address of this stream in its internal state. Keep it in separately
|
||||
// allocated storage so that its address remains stable for the entire compression lifecycle.
|
||||
private let stream: UnsafeMutablePointer<cnioextras_z_stream>
|
||||
var isActive = false
|
||||
|
||||
init() {}
|
||||
// Internal for lifecycle tests: zlib clears this pointer only after deflateEnd releases
|
||||
// the state it allocated for the stream.
|
||||
var zlibStateIsAllocated: Bool {
|
||||
self.stream.pointee.state != nil
|
||||
}
|
||||
|
||||
init() {
|
||||
self.stream = .allocate(capacity: 1)
|
||||
self.stream.initialize(to: cnioextras_z_stream())
|
||||
}
|
||||
|
||||
deinit {
|
||||
self.shutdownIfActive()
|
||||
self.stream.deinitialize(count: 1)
|
||||
self.stream.deallocate()
|
||||
}
|
||||
|
||||
/// Set up the encoder for compressing data according to a specific
|
||||
/// algorithm.
|
||||
mutating func initialize(encoding: Algorithm) {
|
||||
func initialize(encoding: Algorithm) {
|
||||
assert(!isActive)
|
||||
isActive = true
|
||||
// zlib docs say: The application must initialize zalloc, zfree and opaque before calling the init function.
|
||||
stream.zalloc = nil
|
||||
stream.zfree = nil
|
||||
stream.opaque = nil
|
||||
stream.pointee.zalloc = nil
|
||||
stream.pointee.zfree = nil
|
||||
stream.pointee.opaque = nil
|
||||
|
||||
let windowBits: Int32
|
||||
switch encoding.algorithm {
|
||||
@@ -78,7 +94,7 @@ public enum NIOCompression: Sendable {
|
||||
}
|
||||
|
||||
let rc = CNIOExtrasZlib_deflateInit2(
|
||||
&stream,
|
||||
stream,
|
||||
CNIOEXTRAS_Z_DEFAULT_COMPRESSION,
|
||||
CNIOEXTRAS_Z_DEFLATED,
|
||||
windowBits,
|
||||
@@ -86,9 +102,10 @@ public enum NIOCompression: Sendable {
|
||||
CNIOEXTRAS_Z_DEFAULT_STRATEGY
|
||||
)
|
||||
precondition(rc == CNIOEXTRAS_Z_OK, "Unexpected return from zlib init: \(rc)")
|
||||
isActive = true
|
||||
}
|
||||
|
||||
mutating func compress(
|
||||
func compress(
|
||||
inputBuffer: inout ByteBuffer,
|
||||
allocator: ByteBufferAllocator,
|
||||
finalise: Bool
|
||||
@@ -107,37 +124,39 @@ public enum NIOCompression: Sendable {
|
||||
// some compression algorithms and so it should be used only when necessary. This completes the current deflate block and
|
||||
// follows it with an empty stored block that is three bits plus filler bits to the next byte, followed by four bytes
|
||||
// (00 00 ff ff).
|
||||
let bufferSize = Int(cnioextras_z_deflateBound(&stream, UInt(inputBuffer.readableBytes)))
|
||||
let bufferSize = Int(cnioextras_z_deflateBound(stream, UInt(inputBuffer.readableBytes)))
|
||||
var outputBuffer = allocator.buffer(capacity: bufferSize + 5)
|
||||
stream.oneShotDeflate(from: &inputBuffer, to: &outputBuffer, flag: flags)
|
||||
return outputBuffer
|
||||
}
|
||||
|
||||
mutating func shutdown() {
|
||||
@discardableResult
|
||||
func shutdown() -> Int32 {
|
||||
assert(isActive)
|
||||
isActive = false
|
||||
cnioextras_z_deflateEnd(&stream)
|
||||
return cnioextras_z_deflateEnd(stream)
|
||||
}
|
||||
|
||||
mutating func shutdownIfActive() {
|
||||
@discardableResult
|
||||
func shutdownIfActive() -> Int32? {
|
||||
if isActive {
|
||||
isActive = false
|
||||
cnioextras_z_deflateEnd(&stream)
|
||||
return self.shutdown()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension cnioextras_z_stream {
|
||||
extension UnsafeMutablePointer where Pointee == cnioextras_z_stream {
|
||||
/// Executes deflate from one buffer to another buffer. The advantage of this method is that it
|
||||
/// will ensure that the stream is "safe" after each call (that is, that the stream does not have
|
||||
/// pointers to byte buffers any longer).
|
||||
mutating func oneShotDeflate(from: inout ByteBuffer, to: inout ByteBuffer, flag: Int32) {
|
||||
func oneShotDeflate(from: inout ByteBuffer, to: inout ByteBuffer, flag: Int32) {
|
||||
defer {
|
||||
self.avail_in = 0
|
||||
self.next_in = nil
|
||||
self.avail_out = 0
|
||||
self.next_out = nil
|
||||
self.pointee.avail_in = 0
|
||||
self.pointee.next_in = nil
|
||||
self.pointee.avail_out = 0
|
||||
self.pointee.next_out = nil
|
||||
}
|
||||
|
||||
from.readWithUnsafeMutableReadableBytes { dataPtr in
|
||||
@@ -147,20 +166,20 @@ extension cnioextras_z_stream {
|
||||
count: dataPtr.count
|
||||
)
|
||||
|
||||
self.avail_in = UInt32(typedDataPtr.count)
|
||||
self.next_in = typedDataPtr.baseAddress!
|
||||
self.pointee.avail_in = UInt32(typedDataPtr.count)
|
||||
self.pointee.next_in = typedDataPtr.baseAddress!
|
||||
|
||||
let rc = deflateToBuffer(buffer: &to, flag: flag)
|
||||
precondition(rc == CNIOEXTRAS_Z_OK || rc == CNIOEXTRAS_Z_STREAM_END, "One-shot compression failed: \(rc)")
|
||||
|
||||
return typedDataPtr.count - Int(self.avail_in)
|
||||
return typedDataPtr.count - Int(self.pointee.avail_in)
|
||||
}
|
||||
}
|
||||
|
||||
/// A private function that sets the deflate target buffer and then calls deflate.
|
||||
/// This relies on having the input set by the previous caller: it will use whatever input was
|
||||
/// configured.
|
||||
private mutating func deflateToBuffer(buffer: inout ByteBuffer, flag: Int32) -> Int32 {
|
||||
private func deflateToBuffer(buffer: inout ByteBuffer, flag: Int32) -> Int32 {
|
||||
var rc = CNIOEXTRAS_Z_OK
|
||||
|
||||
buffer.writeWithUnsafeMutableBytes(minimumWritableBytes: buffer.capacity) { outputPtr in
|
||||
@@ -168,10 +187,10 @@ extension cnioextras_z_stream {
|
||||
start: outputPtr.baseAddress!.assumingMemoryBound(to: UInt8.self),
|
||||
count: outputPtr.count
|
||||
)
|
||||
self.avail_out = UInt32(typedOutputPtr.count)
|
||||
self.next_out = typedOutputPtr.baseAddress!
|
||||
rc = cnioextras_z_deflate(&self, flag)
|
||||
return typedOutputPtr.count - Int(self.avail_out)
|
||||
self.pointee.avail_out = UInt32(typedOutputPtr.count)
|
||||
self.pointee.next_out = typedOutputPtr.baseAddress!
|
||||
rc = cnioextras_z_deflate(self, flag)
|
||||
return typedOutputPtr.count - Int(self.pointee.avail_out)
|
||||
}
|
||||
|
||||
return rc
|
||||
|
||||
@@ -96,7 +96,7 @@ public enum NIOHTTPDecompression: Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
struct Decompressor {
|
||||
final class Decompressor {
|
||||
/// `15` is the base two logarithm of the maximum window size (the size of the history buffer). It should be in the range 8..15 for this version of the library.
|
||||
/// `32` enables automatic detection of gzip or zlib compression format with automatic header detection.
|
||||
///
|
||||
@@ -130,14 +130,31 @@ public enum NIOHTTPDecompression: Sendable {
|
||||
static let windowBitsWithAutomaticCompressionFormatDetection: Int32 = 15 + 32
|
||||
|
||||
private let limit: NIOHTTPDecompression.DecompressionLimit
|
||||
private var stream = cnioextras_z_stream()
|
||||
// zlib stores the address of this stream in its internal state. Keep it in separately
|
||||
// allocated storage so that its address remains stable for the entire decompression lifecycle.
|
||||
private let stream: UnsafeMutablePointer<cnioextras_z_stream>
|
||||
private var inflated = 0
|
||||
private var isInitialized = false
|
||||
|
||||
// Internal for lifecycle tests: zlib clears this pointer only after inflateEnd releases
|
||||
// the state it allocated for the stream.
|
||||
var zlibStateIsAllocated: Bool {
|
||||
self.stream.pointee.state != nil
|
||||
}
|
||||
|
||||
init(limit: NIOHTTPDecompression.DecompressionLimit) {
|
||||
self.limit = limit
|
||||
self.stream = .allocate(capacity: 1)
|
||||
self.stream.initialize(to: cnioextras_z_stream())
|
||||
}
|
||||
|
||||
mutating func decompress(
|
||||
deinit {
|
||||
self.deinitializeDecoder()
|
||||
self.stream.deinitialize(count: 1)
|
||||
self.stream.deallocate()
|
||||
}
|
||||
|
||||
func decompress(
|
||||
part: inout ByteBuffer,
|
||||
buffer: inout ByteBuffer,
|
||||
compressedLength: Int
|
||||
@@ -152,60 +169,71 @@ public enum NIOHTTPDecompression: Sendable {
|
||||
return result
|
||||
}
|
||||
|
||||
mutating func initializeDecoder() throws {
|
||||
self.stream.zalloc = nil
|
||||
self.stream.zfree = nil
|
||||
self.stream.opaque = nil
|
||||
func initializeDecoder() throws {
|
||||
try self.initializeDecoder(windowBits: Self.windowBitsWithAutomaticCompressionFormatDetection)
|
||||
}
|
||||
|
||||
func initializeDecoder(windowBits: Int32) throws {
|
||||
precondition(!self.isInitialized)
|
||||
self.stream.pointee.zalloc = nil
|
||||
self.stream.pointee.zfree = nil
|
||||
self.stream.pointee.opaque = nil
|
||||
self.inflated = 0
|
||||
|
||||
let rc = CNIOExtrasZlib_inflateInit2(&self.stream, Self.windowBitsWithAutomaticCompressionFormatDetection)
|
||||
let rc = CNIOExtrasZlib_inflateInit2(self.stream, windowBits)
|
||||
guard rc == CNIOEXTRAS_Z_OK else {
|
||||
throw NIOHTTPDecompression.DecompressionError.initializationError(Int(rc))
|
||||
}
|
||||
self.isInitialized = true
|
||||
}
|
||||
|
||||
mutating func deinitializeDecoder() {
|
||||
cnioextras_z_inflateEnd(&self.stream)
|
||||
@discardableResult
|
||||
func deinitializeDecoder() -> Int32? {
|
||||
if self.isInitialized {
|
||||
self.isInitialized = false
|
||||
return cnioextras_z_inflateEnd(self.stream)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension cnioextras_z_stream {
|
||||
mutating func inflatePart(input: inout ByteBuffer, output: inout ByteBuffer) throws -> InflateResult {
|
||||
extension UnsafeMutablePointer where Pointee == cnioextras_z_stream {
|
||||
func inflatePart(input: inout ByteBuffer, output: inout ByteBuffer) throws -> InflateResult {
|
||||
let minimumCapacity = input.readableBytes * 2
|
||||
var inflateResult = InflateResult(written: 0, complete: false)
|
||||
|
||||
try input.readWithUnsafeMutableReadableBytes { pointer in
|
||||
self.avail_in = UInt32(pointer.count)
|
||||
self.next_in = CNIOExtrasZlib_voidPtr_to_BytefPtr(pointer.baseAddress!)
|
||||
self.pointee.avail_in = UInt32(pointer.count)
|
||||
self.pointee.next_in = CNIOExtrasZlib_voidPtr_to_BytefPtr(pointer.baseAddress!)
|
||||
|
||||
defer {
|
||||
self.avail_in = 0
|
||||
self.next_in = nil
|
||||
self.avail_out = 0
|
||||
self.next_out = nil
|
||||
self.pointee.avail_in = 0
|
||||
self.pointee.next_in = nil
|
||||
self.pointee.avail_out = 0
|
||||
self.pointee.next_out = nil
|
||||
}
|
||||
|
||||
inflateResult = try self.inflatePart(to: &output, minimumCapacity: minimumCapacity)
|
||||
|
||||
return pointer.count - Int(self.avail_in)
|
||||
return pointer.count - Int(self.pointee.avail_in)
|
||||
}
|
||||
return inflateResult
|
||||
}
|
||||
|
||||
private mutating func inflatePart(to buffer: inout ByteBuffer, minimumCapacity: Int) throws -> InflateResult {
|
||||
private func inflatePart(to buffer: inout ByteBuffer, minimumCapacity: Int) throws -> InflateResult {
|
||||
var rc = CNIOEXTRAS_Z_OK
|
||||
|
||||
let written = try buffer.writeWithUnsafeMutableBytes(minimumWritableBytes: minimumCapacity) { pointer in
|
||||
self.avail_out = UInt32(pointer.count)
|
||||
self.next_out = CNIOExtrasZlib_voidPtr_to_BytefPtr(pointer.baseAddress!)
|
||||
self.pointee.avail_out = UInt32(pointer.count)
|
||||
self.pointee.next_out = CNIOExtrasZlib_voidPtr_to_BytefPtr(pointer.baseAddress!)
|
||||
|
||||
rc = cnioextras_z_inflate(&self, CNIOEXTRAS_Z_NO_FLUSH)
|
||||
rc = cnioextras_z_inflate(self, CNIOEXTRAS_Z_NO_FLUSH)
|
||||
guard rc == CNIOEXTRAS_Z_OK || rc == CNIOEXTRAS_Z_STREAM_END else {
|
||||
throw NIOHTTPDecompression.DecompressionError.inflationError(Int(rc))
|
||||
}
|
||||
|
||||
return pointer.count - Int(self.avail_out)
|
||||
return pointer.count - Int(self.pointee.avail_out)
|
||||
}
|
||||
|
||||
return InflateResult(written: written, complete: rc == CNIOEXTRAS_Z_STREAM_END)
|
||||
|
||||
@@ -98,7 +98,7 @@ private final class HTTPDecompressionDecoder: NIOSingleStepByteToMessageDecoder
|
||||
typealias InboundOut = ByteBuffer
|
||||
|
||||
private let allocator: ByteBufferAllocator
|
||||
private var decompressor: NIOHTTPDecompression.Decompressor
|
||||
private let decompressor: NIOHTTPDecompression.Decompressor
|
||||
/// The number of already consumed compressed bytes
|
||||
private var compressedLength: Int
|
||||
private var complete: Bool
|
||||
|
||||
@@ -51,7 +51,7 @@ public final class NIOHTTPRequestCompressor: ChannelOutboundHandler, RemovableCh
|
||||
/// handler state
|
||||
var state: State
|
||||
/// compression handler
|
||||
var compressor: NIOCompression.Compressor
|
||||
let compressor: NIOCompression.Compressor
|
||||
/// pending write promise
|
||||
var pendingWritePromise: EventLoopPromise<Void>!
|
||||
|
||||
|
||||
@@ -126,7 +126,7 @@ public final class HTTPResponseCompressor: ChannelDuplexHandler, RemovableChanne
|
||||
case noDataToWrite
|
||||
}
|
||||
|
||||
private var compressor: NIOCompression.Compressor
|
||||
private let compressor: NIOCompression.Compressor
|
||||
|
||||
// A queue of accept headers.
|
||||
private var acceptQueue = CircularBuffer<[Substring]>(initialCapacity: 8)
|
||||
@@ -270,7 +270,7 @@ public final class HTTPResponseCompressor: ChannelDuplexHandler, RemovableChanne
|
||||
///
|
||||
/// Called either when a HTTP end message is received or our flush() method is called.
|
||||
private func emitPendingWrites(context: ChannelHandlerContext) {
|
||||
let writesToEmit = pendingResponse.flush(compressor: &compressor, allocator: context.channel.allocator)
|
||||
let writesToEmit = pendingResponse.flush(compressor: compressor, allocator: context.channel.allocator)
|
||||
var pendingPromise = pendingWritePromise
|
||||
|
||||
if let writeHead = writesToEmit.0 {
|
||||
@@ -372,7 +372,7 @@ private struct PartialHTTPResponse {
|
||||
/// Calling this function resets the buffer, freeing any excess memory allocated in the internal
|
||||
/// buffer and losing all copies of the other HTTP data. At this point it may freely be reused.
|
||||
mutating func flush(
|
||||
compressor: inout NIOCompression.Compressor,
|
||||
compressor: NIOCompression.Compressor,
|
||||
allocator: ByteBufferAllocator
|
||||
) -> (HTTPResponseHead?, ByteBuffer?, HTTPServerResponsePart?) {
|
||||
var outputBody: ByteBuffer? = nil
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the SwiftNIO open source project
|
||||
//
|
||||
// Copyright (c) 2026 Apple Inc. and the SwiftNIO project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
import CNIOExtrasZlib
|
||||
import NIOCore
|
||||
import XCTest
|
||||
|
||||
@testable import NIOHTTPCompression
|
||||
|
||||
final class ZlibStreamLifecycleTest: XCTestCase {
|
||||
func testDecompressorCanInitializeAfterInitializationFailure() throws {
|
||||
let decompressor = NIOHTTPDecompression.Decompressor(limit: .none)
|
||||
|
||||
XCTAssertThrowsError(try decompressor.initializeDecoder(windowBits: 100)) { error in
|
||||
XCTAssertEqual(
|
||||
error as? NIOHTTPDecompression.DecompressionError,
|
||||
.initializationError(Int(CNIOEXTRAS_Z_STREAM_ERROR))
|
||||
)
|
||||
}
|
||||
|
||||
XCTAssertNoThrow(try decompressor.initializeDecoder())
|
||||
XCTAssertEqual(decompressor.deinitializeDecoder(), CNIOEXTRAS_Z_OK)
|
||||
// Explicit deinitialization is idempotent so the stream owner cannot call inflateEnd twice.
|
||||
XCTAssertNil(decompressor.deinitializeDecoder())
|
||||
}
|
||||
|
||||
func testStreamOwnersSupportMultipleCompressionLifecycles() throws {
|
||||
let allocator = ByteBufferAllocator()
|
||||
let compressor = NIOCompression.Compressor()
|
||||
let decompressor = NIOHTTPDecompression.Decompressor(limit: .none)
|
||||
|
||||
for (encoding, chunks) in [
|
||||
(NIOCompression.Algorithm.gzip, ["first compression ", "lifecycle"]),
|
||||
(NIOCompression.Algorithm.deflate, ["second compression ", "lifecycle"]),
|
||||
] {
|
||||
compressor.initialize(encoding: encoding)
|
||||
XCTAssertTrue(compressor.zlibStateIsAllocated)
|
||||
var firstInput = allocator.buffer(capacity: chunks[0].utf8.count)
|
||||
firstInput.writeString(chunks[0])
|
||||
var compressed = compressor.compress(inputBuffer: &firstInput, allocator: allocator, finalise: false)
|
||||
|
||||
var secondInput = allocator.buffer(capacity: chunks[1].utf8.count)
|
||||
secondInput.writeString(chunks[1])
|
||||
var finalCompressed = compressor.compress(inputBuffer: &secondInput, allocator: allocator, finalise: true)
|
||||
compressed.writeBuffer(&finalCompressed)
|
||||
XCTAssertEqual(compressor.shutdown(), CNIOEXTRAS_Z_OK)
|
||||
XCTAssertNil(compressor.shutdownIfActive())
|
||||
XCTAssertFalse(compressor.zlibStateIsAllocated)
|
||||
|
||||
let text = chunks.joined()
|
||||
let compressedLength = compressed.readableBytes
|
||||
var firstCompressedPart = compressed.readSlice(length: compressedLength / 2)!
|
||||
var output = allocator.buffer(capacity: text.utf8.count)
|
||||
try decompressor.initializeDecoder()
|
||||
XCTAssertTrue(decompressor.zlibStateIsAllocated)
|
||||
let firstResult = try decompressor.decompress(
|
||||
part: &firstCompressedPart,
|
||||
buffer: &output,
|
||||
compressedLength: compressedLength
|
||||
)
|
||||
let finalResult = try decompressor.decompress(
|
||||
part: &compressed,
|
||||
buffer: &output,
|
||||
compressedLength: compressedLength
|
||||
)
|
||||
XCTAssertEqual(decompressor.deinitializeDecoder(), CNIOEXTRAS_Z_OK)
|
||||
XCTAssertFalse(decompressor.zlibStateIsAllocated)
|
||||
|
||||
XCTAssertFalse(firstResult.complete)
|
||||
XCTAssertTrue(finalResult.complete)
|
||||
XCTAssertEqual(firstCompressedPart.readableBytes, 0)
|
||||
XCTAssertEqual(compressed.readableBytes, 0)
|
||||
XCTAssertEqual(output.readString(length: output.readableBytes), text)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in new issue
Block a user