Keep zlib streams at stable addresses (#321)

### Motivation:

zlib stores the address of `z_stream` in its internal state and expects
that
address to remain stable across calls. Storing `z_stream` inline in
movable
Swift value types does not provide that guarantee and can cause
`Z_STREAM_ERROR` during cleanup.

Fixes #308.

### Modifications:

- Make the internal compressor and decompressor reference owners of
separately
  allocated `z_stream` storage.
- Pass the stable pointer to every zlib initialization, processing, and
cleanup
  call.
- Keep lifecycle state with the pointer owner and make conditional
cleanup
  idempotent.
- Add regression tests covering failed initialization recovery, multiple
compression and decompression calls, repeated lifecycles, and
deallocation
  cleanup.

### Result:

zlib observes the same `z_stream` address for each complete lifecycle,
while
stream cleanup remains single-owner and safe against repeated shutdown.
This commit is contained in:
Tianhao Liu authored and GitHub committed 2026-08-06 07:10:41 +00:00
1 parent aedcd7e2d4
commit 4ba1f167a7
6 files changed
+194 -59

No files matched your search

@@ -53,21 +53,37 @@ public enum NIOCompression: Sendable {
}
/// Data compression utility.
struct Compressor {
private var stream = cnioextras_z_stream()
final class Compressor {
// zlib stores the address of this stream in its internal state. Keep it in separately
// allocated storage so that its address remains stable for the entire compression lifecycle.
private let stream: UnsafeMutablePointer<cnioextras_z_stream>
var isActive = false
init() {}
// Internal for lifecycle tests: zlib clears this pointer only after deflateEnd releases
// the state it allocated for the stream.
var zlibStateIsAllocated: Bool {
self.stream.pointee.state != nil
}
init() {
self.stream = .allocate(capacity: 1)
self.stream.initialize(to: cnioextras_z_stream())
}
deinit {
self.shutdownIfActive()
self.stream.deinitialize(count: 1)
self.stream.deallocate()
}
/// Set up the encoder for compressing data according to a specific
/// algorithm.
mutating func initialize(encoding: Algorithm) {
func initialize(encoding: Algorithm) {
assert(!isActive)
isActive = true
// zlib docs say: The application must initialize zalloc, zfree and opaque before calling the init function.
stream.zalloc = nil
stream.zfree = nil
stream.opaque = nil
stream.pointee.zalloc = nil
stream.pointee.zfree = nil
stream.pointee.opaque = nil
let windowBits: Int32
switch encoding.algorithm {
@@ -78,7 +94,7 @@ public enum NIOCompression: Sendable {
}
let rc = CNIOExtrasZlib_deflateInit2(
&stream,
stream,
CNIOEXTRAS_Z_DEFAULT_COMPRESSION,
CNIOEXTRAS_Z_DEFLATED,
windowBits,
@@ -86,9 +102,10 @@ public enum NIOCompression: Sendable {
CNIOEXTRAS_Z_DEFAULT_STRATEGY
)
precondition(rc == CNIOEXTRAS_Z_OK, "Unexpected return from zlib init: \(rc)")
isActive = true
}
mutating func compress(
func compress(
inputBuffer: inout ByteBuffer,
allocator: ByteBufferAllocator,
finalise: Bool
@@ -107,37 +124,39 @@ public enum NIOCompression: Sendable {
// some compression algorithms and so it should be used only when necessary. This completes the current deflate block and
// follows it with an empty stored block that is three bits plus filler bits to the next byte, followed by four bytes
// (00 00 ff ff).
let bufferSize = Int(cnioextras_z_deflateBound(&stream, UInt(inputBuffer.readableBytes)))
let bufferSize = Int(cnioextras_z_deflateBound(stream, UInt(inputBuffer.readableBytes)))
var outputBuffer = allocator.buffer(capacity: bufferSize + 5)
stream.oneShotDeflate(from: &inputBuffer, to: &outputBuffer, flag: flags)
return outputBuffer
}
mutating func shutdown() {
@discardableResult
func shutdown() -> Int32 {
assert(isActive)
isActive = false
cnioextras_z_deflateEnd(&stream)
return cnioextras_z_deflateEnd(stream)
}
mutating func shutdownIfActive() {
@discardableResult
func shutdownIfActive() -> Int32? {
if isActive {
isActive = false
cnioextras_z_deflateEnd(&stream)
return self.shutdown()
}
return nil
}
}
}
extension cnioextras_z_stream {
extension UnsafeMutablePointer where Pointee == cnioextras_z_stream {
/// Executes deflate from one buffer to another buffer. The advantage of this method is that it
/// will ensure that the stream is "safe" after each call (that is, that the stream does not have
/// pointers to byte buffers any longer).
mutating func oneShotDeflate(from: inout ByteBuffer, to: inout ByteBuffer, flag: Int32) {
func oneShotDeflate(from: inout ByteBuffer, to: inout ByteBuffer, flag: Int32) {
defer {
self.avail_in = 0
self.next_in = nil
self.avail_out = 0
self.next_out = nil
self.pointee.avail_in = 0
self.pointee.next_in = nil
self.pointee.avail_out = 0
self.pointee.next_out = nil
}
from.readWithUnsafeMutableReadableBytes { dataPtr in
@@ -147,20 +166,20 @@ extension cnioextras_z_stream {
count: dataPtr.count
)
self.avail_in = UInt32(typedDataPtr.count)
self.next_in = typedDataPtr.baseAddress!
self.pointee.avail_in = UInt32(typedDataPtr.count)
self.pointee.next_in = typedDataPtr.baseAddress!
let rc = deflateToBuffer(buffer: &to, flag: flag)
precondition(rc == CNIOEXTRAS_Z_OK || rc == CNIOEXTRAS_Z_STREAM_END, "One-shot compression failed: \(rc)")
return typedDataPtr.count - Int(self.avail_in)
return typedDataPtr.count - Int(self.pointee.avail_in)
}
}
/// A private function that sets the deflate target buffer and then calls deflate.
/// This relies on having the input set by the previous caller: it will use whatever input was
/// configured.
private mutating func deflateToBuffer(buffer: inout ByteBuffer, flag: Int32) -> Int32 {
private func deflateToBuffer(buffer: inout ByteBuffer, flag: Int32) -> Int32 {
var rc = CNIOEXTRAS_Z_OK
buffer.writeWithUnsafeMutableBytes(minimumWritableBytes: buffer.capacity) { outputPtr in
@@ -168,10 +187,10 @@ extension cnioextras_z_stream {
start: outputPtr.baseAddress!.assumingMemoryBound(to: UInt8.self),
count: outputPtr.count
)
self.avail_out = UInt32(typedOutputPtr.count)
self.next_out = typedOutputPtr.baseAddress!
rc = cnioextras_z_deflate(&self, flag)
return typedOutputPtr.count - Int(self.avail_out)
self.pointee.avail_out = UInt32(typedOutputPtr.count)
self.pointee.next_out = typedOutputPtr.baseAddress!
rc = cnioextras_z_deflate(self, flag)
return typedOutputPtr.count - Int(self.pointee.avail_out)
}
return rc
@@ -96,7 +96,7 @@ public enum NIOHTTPDecompression: Sendable {
}
}
struct Decompressor {
final class Decompressor {
/// `15` is the base two logarithm of the maximum window size (the size of the history buffer). It should be in the range 8..15 for this version of the library.
/// `32` enables automatic detection of gzip or zlib compression format with automatic header detection.
///
@@ -130,14 +130,31 @@ public enum NIOHTTPDecompression: Sendable {
static let windowBitsWithAutomaticCompressionFormatDetection: Int32 = 15 + 32
private let limit: NIOHTTPDecompression.DecompressionLimit
private var stream = cnioextras_z_stream()
// zlib stores the address of this stream in its internal state. Keep it in separately
// allocated storage so that its address remains stable for the entire decompression lifecycle.
private let stream: UnsafeMutablePointer<cnioextras_z_stream>
private var inflated = 0
private var isInitialized = false
// Internal for lifecycle tests: zlib clears this pointer only after inflateEnd releases
// the state it allocated for the stream.
var zlibStateIsAllocated: Bool {
self.stream.pointee.state != nil
}
init(limit: NIOHTTPDecompression.DecompressionLimit) {
self.limit = limit
self.stream = .allocate(capacity: 1)
self.stream.initialize(to: cnioextras_z_stream())
}
mutating func decompress(
deinit {
self.deinitializeDecoder()
self.stream.deinitialize(count: 1)
self.stream.deallocate()
}
func decompress(
part: inout ByteBuffer,
buffer: inout ByteBuffer,
compressedLength: Int
@@ -152,60 +169,71 @@ public enum NIOHTTPDecompression: Sendable {
return result
}
mutating func initializeDecoder() throws {
self.stream.zalloc = nil
self.stream.zfree = nil
self.stream.opaque = nil
func initializeDecoder() throws {
try self.initializeDecoder(windowBits: Self.windowBitsWithAutomaticCompressionFormatDetection)
}
func initializeDecoder(windowBits: Int32) throws {
precondition(!self.isInitialized)
self.stream.pointee.zalloc = nil
self.stream.pointee.zfree = nil
self.stream.pointee.opaque = nil
self.inflated = 0
let rc = CNIOExtrasZlib_inflateInit2(&self.stream, Self.windowBitsWithAutomaticCompressionFormatDetection)
let rc = CNIOExtrasZlib_inflateInit2(self.stream, windowBits)
guard rc == CNIOEXTRAS_Z_OK else {
throw NIOHTTPDecompression.DecompressionError.initializationError(Int(rc))
}
self.isInitialized = true
}
mutating func deinitializeDecoder() {
cnioextras_z_inflateEnd(&self.stream)
@discardableResult
func deinitializeDecoder() -> Int32? {
if self.isInitialized {
self.isInitialized = false
return cnioextras_z_inflateEnd(self.stream)
}
return nil
}
}
}
extension cnioextras_z_stream {
mutating func inflatePart(input: inout ByteBuffer, output: inout ByteBuffer) throws -> InflateResult {
extension UnsafeMutablePointer where Pointee == cnioextras_z_stream {
func inflatePart(input: inout ByteBuffer, output: inout ByteBuffer) throws -> InflateResult {
let minimumCapacity = input.readableBytes * 2
var inflateResult = InflateResult(written: 0, complete: false)
try input.readWithUnsafeMutableReadableBytes { pointer in
self.avail_in = UInt32(pointer.count)
self.next_in = CNIOExtrasZlib_voidPtr_to_BytefPtr(pointer.baseAddress!)
self.pointee.avail_in = UInt32(pointer.count)
self.pointee.next_in = CNIOExtrasZlib_voidPtr_to_BytefPtr(pointer.baseAddress!)
defer {
self.avail_in = 0
self.next_in = nil
self.avail_out = 0
self.next_out = nil
self.pointee.avail_in = 0
self.pointee.next_in = nil
self.pointee.avail_out = 0
self.pointee.next_out = nil
}
inflateResult = try self.inflatePart(to: &output, minimumCapacity: minimumCapacity)
return pointer.count - Int(self.avail_in)
return pointer.count - Int(self.pointee.avail_in)
}
return inflateResult
}
private mutating func inflatePart(to buffer: inout ByteBuffer, minimumCapacity: Int) throws -> InflateResult {
private func inflatePart(to buffer: inout ByteBuffer, minimumCapacity: Int) throws -> InflateResult {
var rc = CNIOEXTRAS_Z_OK
let written = try buffer.writeWithUnsafeMutableBytes(minimumWritableBytes: minimumCapacity) { pointer in
self.avail_out = UInt32(pointer.count)
self.next_out = CNIOExtrasZlib_voidPtr_to_BytefPtr(pointer.baseAddress!)
self.pointee.avail_out = UInt32(pointer.count)
self.pointee.next_out = CNIOExtrasZlib_voidPtr_to_BytefPtr(pointer.baseAddress!)
rc = cnioextras_z_inflate(&self, CNIOEXTRAS_Z_NO_FLUSH)
rc = cnioextras_z_inflate(self, CNIOEXTRAS_Z_NO_FLUSH)
guard rc == CNIOEXTRAS_Z_OK || rc == CNIOEXTRAS_Z_STREAM_END else {
throw NIOHTTPDecompression.DecompressionError.inflationError(Int(rc))
}
return pointer.count - Int(self.avail_out)
return pointer.count - Int(self.pointee.avail_out)
}
return InflateResult(written: written, complete: rc == CNIOEXTRAS_Z_STREAM_END)
@@ -98,7 +98,7 @@ private final class HTTPDecompressionDecoder: NIOSingleStepByteToMessageDecoder
typealias InboundOut = ByteBuffer
private let allocator: ByteBufferAllocator
private var decompressor: NIOHTTPDecompression.Decompressor
private let decompressor: NIOHTTPDecompression.Decompressor
/// The number of already consumed compressed bytes
private var compressedLength: Int
private var complete: Bool
@@ -51,7 +51,7 @@ public final class NIOHTTPRequestCompressor: ChannelOutboundHandler, RemovableCh
/// handler state
var state: State
/// compression handler
var compressor: NIOCompression.Compressor
let compressor: NIOCompression.Compressor
/// pending write promise
var pendingWritePromise: EventLoopPromise<Void>!
@@ -126,7 +126,7 @@ public final class HTTPResponseCompressor: ChannelDuplexHandler, RemovableChanne
case noDataToWrite
}
private var compressor: NIOCompression.Compressor
private let compressor: NIOCompression.Compressor
// A queue of accept headers.
private var acceptQueue = CircularBuffer<[Substring]>(initialCapacity: 8)
@@ -270,7 +270,7 @@ public final class HTTPResponseCompressor: ChannelDuplexHandler, RemovableChanne
///
/// Called either when a HTTP end message is received or our flush() method is called.
private func emitPendingWrites(context: ChannelHandlerContext) {
let writesToEmit = pendingResponse.flush(compressor: &compressor, allocator: context.channel.allocator)
let writesToEmit = pendingResponse.flush(compressor: compressor, allocator: context.channel.allocator)
var pendingPromise = pendingWritePromise
if let writeHead = writesToEmit.0 {
@@ -372,7 +372,7 @@ private struct PartialHTTPResponse {
/// Calling this function resets the buffer, freeing any excess memory allocated in the internal
/// buffer and losing all copies of the other HTTP data. At this point it may freely be reused.
mutating func flush(
compressor: inout NIOCompression.Compressor,
compressor: NIOCompression.Compressor,
allocator: ByteBufferAllocator
) -> (HTTPResponseHead?, ByteBuffer?, HTTPServerResponsePart?) {
var outputBody: ByteBuffer? = nil
@@ -0,0 +1,88 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2026 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
import CNIOExtrasZlib
import NIOCore
import XCTest
@testable import NIOHTTPCompression
final class ZlibStreamLifecycleTest: XCTestCase {
func testDecompressorCanInitializeAfterInitializationFailure() throws {
let decompressor = NIOHTTPDecompression.Decompressor(limit: .none)
XCTAssertThrowsError(try decompressor.initializeDecoder(windowBits: 100)) { error in
XCTAssertEqual(
error as? NIOHTTPDecompression.DecompressionError,
.initializationError(Int(CNIOEXTRAS_Z_STREAM_ERROR))
)
}
XCTAssertNoThrow(try decompressor.initializeDecoder())
XCTAssertEqual(decompressor.deinitializeDecoder(), CNIOEXTRAS_Z_OK)
// Explicit deinitialization is idempotent so the stream owner cannot call inflateEnd twice.
XCTAssertNil(decompressor.deinitializeDecoder())
}
func testStreamOwnersSupportMultipleCompressionLifecycles() throws {
let allocator = ByteBufferAllocator()
let compressor = NIOCompression.Compressor()
let decompressor = NIOHTTPDecompression.Decompressor(limit: .none)
for (encoding, chunks) in [
(NIOCompression.Algorithm.gzip, ["first compression ", "lifecycle"]),
(NIOCompression.Algorithm.deflate, ["second compression ", "lifecycle"]),
] {
compressor.initialize(encoding: encoding)
XCTAssertTrue(compressor.zlibStateIsAllocated)
var firstInput = allocator.buffer(capacity: chunks[0].utf8.count)
firstInput.writeString(chunks[0])
var compressed = compressor.compress(inputBuffer: &firstInput, allocator: allocator, finalise: false)
var secondInput = allocator.buffer(capacity: chunks[1].utf8.count)
secondInput.writeString(chunks[1])
var finalCompressed = compressor.compress(inputBuffer: &secondInput, allocator: allocator, finalise: true)
compressed.writeBuffer(&finalCompressed)
XCTAssertEqual(compressor.shutdown(), CNIOEXTRAS_Z_OK)
XCTAssertNil(compressor.shutdownIfActive())
XCTAssertFalse(compressor.zlibStateIsAllocated)
let text = chunks.joined()
let compressedLength = compressed.readableBytes
var firstCompressedPart = compressed.readSlice(length: compressedLength / 2)!
var output = allocator.buffer(capacity: text.utf8.count)
try decompressor.initializeDecoder()
XCTAssertTrue(decompressor.zlibStateIsAllocated)
let firstResult = try decompressor.decompress(
part: &firstCompressedPart,
buffer: &output,
compressedLength: compressedLength
)
let finalResult = try decompressor.decompress(
part: &compressed,
buffer: &output,
compressedLength: compressedLength
)
XCTAssertEqual(decompressor.deinitializeDecoder(), CNIOEXTRAS_Z_OK)
XCTAssertFalse(decompressor.zlibStateIsAllocated)
XCTAssertFalse(firstResult.complete)
XCTAssertTrue(finalResult.complete)
XCTAssertEqual(firstCompressedPart.readableBytes, 0)
XCTAssertEqual(compressed.readableBytes, 0)
XCTAssertEqual(output.readString(length: output.readableBytes), text)
}
}
}