From 4ba1f167a7dfe33cf150ca5d3a221d3f566a03c3 Mon Sep 17 00:00:00 2001 From: Tianhao Liu Date: Thu, 6 Aug 2026 00:10:41 -0700 Subject: [PATCH] Keep zlib streams at stable addresses (#321) ### Motivation: zlib stores the address of `z_stream` in its internal state and expects that address to remain stable across calls. Storing `z_stream` inline in movable Swift value types does not provide that guarantee and can cause `Z_STREAM_ERROR` during cleanup. Fixes #308. ### Modifications: - Make the internal compressor and decompressor reference owners of separately allocated `z_stream` storage. - Pass the stable pointer to every zlib initialization, processing, and cleanup call. - Keep lifecycle state with the pointer owner and make conditional cleanup idempotent. - Add regression tests covering failed initialization recovery, multiple compression and decompression calls, repeated lifecycles, and deallocation cleanup. ### Result: zlib observes the same `z_stream` address for each complete lifecycle, while stream cleanup remains single-owner and safe against repeated shutdown. --- .../NIOHTTPCompression/HTTPCompression.swift | 79 ++++++++++------- .../HTTPDecompression.swift | 76 +++++++++++----- .../NIOHTTPCompression/HTTPDecompressor.swift | 2 +- .../HTTPRequestCompressor.swift | 2 +- .../HTTPResponseCompressor.swift | 6 +- .../ZlibStreamLifecycleTest.swift | 88 +++++++++++++++++++ 6 files changed, 194 insertions(+), 59 deletions(-) create mode 100644 Tests/NIOHTTPCompressionTests/ZlibStreamLifecycleTest.swift diff --git a/Sources/NIOHTTPCompression/HTTPCompression.swift b/Sources/NIOHTTPCompression/HTTPCompression.swift index 8fa3cac..e3bc842 100644 --- a/Sources/NIOHTTPCompression/HTTPCompression.swift +++ b/Sources/NIOHTTPCompression/HTTPCompression.swift @@ -53,21 +53,37 @@ public enum NIOCompression: Sendable { } /// Data compression utility. - struct Compressor { - private var stream = cnioextras_z_stream() + final class Compressor { + // zlib stores the address of this stream in its internal state. Keep it in separately + // allocated storage so that its address remains stable for the entire compression lifecycle. + private let stream: UnsafeMutablePointer var isActive = false - init() {} + // Internal for lifecycle tests: zlib clears this pointer only after deflateEnd releases + // the state it allocated for the stream. + var zlibStateIsAllocated: Bool { + self.stream.pointee.state != nil + } + + init() { + self.stream = .allocate(capacity: 1) + self.stream.initialize(to: cnioextras_z_stream()) + } + + deinit { + self.shutdownIfActive() + self.stream.deinitialize(count: 1) + self.stream.deallocate() + } /// Set up the encoder for compressing data according to a specific /// algorithm. - mutating func initialize(encoding: Algorithm) { + func initialize(encoding: Algorithm) { assert(!isActive) - isActive = true // zlib docs say: The application must initialize zalloc, zfree and opaque before calling the init function. - stream.zalloc = nil - stream.zfree = nil - stream.opaque = nil + stream.pointee.zalloc = nil + stream.pointee.zfree = nil + stream.pointee.opaque = nil let windowBits: Int32 switch encoding.algorithm { @@ -78,7 +94,7 @@ public enum NIOCompression: Sendable { } let rc = CNIOExtrasZlib_deflateInit2( - &stream, + stream, CNIOEXTRAS_Z_DEFAULT_COMPRESSION, CNIOEXTRAS_Z_DEFLATED, windowBits, @@ -86,9 +102,10 @@ public enum NIOCompression: Sendable { CNIOEXTRAS_Z_DEFAULT_STRATEGY ) precondition(rc == CNIOEXTRAS_Z_OK, "Unexpected return from zlib init: \(rc)") + isActive = true } - mutating func compress( + func compress( inputBuffer: inout ByteBuffer, allocator: ByteBufferAllocator, finalise: Bool @@ -107,37 +124,39 @@ public enum NIOCompression: Sendable { // some compression algorithms and so it should be used only when necessary. This completes the current deflate block and // follows it with an empty stored block that is three bits plus filler bits to the next byte, followed by four bytes // (00 00 ff ff). - let bufferSize = Int(cnioextras_z_deflateBound(&stream, UInt(inputBuffer.readableBytes))) + let bufferSize = Int(cnioextras_z_deflateBound(stream, UInt(inputBuffer.readableBytes))) var outputBuffer = allocator.buffer(capacity: bufferSize + 5) stream.oneShotDeflate(from: &inputBuffer, to: &outputBuffer, flag: flags) return outputBuffer } - mutating func shutdown() { + @discardableResult + func shutdown() -> Int32 { assert(isActive) isActive = false - cnioextras_z_deflateEnd(&stream) + return cnioextras_z_deflateEnd(stream) } - mutating func shutdownIfActive() { + @discardableResult + func shutdownIfActive() -> Int32? { if isActive { - isActive = false - cnioextras_z_deflateEnd(&stream) + return self.shutdown() } + return nil } } } -extension cnioextras_z_stream { +extension UnsafeMutablePointer where Pointee == cnioextras_z_stream { /// Executes deflate from one buffer to another buffer. The advantage of this method is that it /// will ensure that the stream is "safe" after each call (that is, that the stream does not have /// pointers to byte buffers any longer). - mutating func oneShotDeflate(from: inout ByteBuffer, to: inout ByteBuffer, flag: Int32) { + func oneShotDeflate(from: inout ByteBuffer, to: inout ByteBuffer, flag: Int32) { defer { - self.avail_in = 0 - self.next_in = nil - self.avail_out = 0 - self.next_out = nil + self.pointee.avail_in = 0 + self.pointee.next_in = nil + self.pointee.avail_out = 0 + self.pointee.next_out = nil } from.readWithUnsafeMutableReadableBytes { dataPtr in @@ -147,20 +166,20 @@ extension cnioextras_z_stream { count: dataPtr.count ) - self.avail_in = UInt32(typedDataPtr.count) - self.next_in = typedDataPtr.baseAddress! + self.pointee.avail_in = UInt32(typedDataPtr.count) + self.pointee.next_in = typedDataPtr.baseAddress! let rc = deflateToBuffer(buffer: &to, flag: flag) precondition(rc == CNIOEXTRAS_Z_OK || rc == CNIOEXTRAS_Z_STREAM_END, "One-shot compression failed: \(rc)") - return typedDataPtr.count - Int(self.avail_in) + return typedDataPtr.count - Int(self.pointee.avail_in) } } /// A private function that sets the deflate target buffer and then calls deflate. /// This relies on having the input set by the previous caller: it will use whatever input was /// configured. - private mutating func deflateToBuffer(buffer: inout ByteBuffer, flag: Int32) -> Int32 { + private func deflateToBuffer(buffer: inout ByteBuffer, flag: Int32) -> Int32 { var rc = CNIOEXTRAS_Z_OK buffer.writeWithUnsafeMutableBytes(minimumWritableBytes: buffer.capacity) { outputPtr in @@ -168,10 +187,10 @@ extension cnioextras_z_stream { start: outputPtr.baseAddress!.assumingMemoryBound(to: UInt8.self), count: outputPtr.count ) - self.avail_out = UInt32(typedOutputPtr.count) - self.next_out = typedOutputPtr.baseAddress! - rc = cnioextras_z_deflate(&self, flag) - return typedOutputPtr.count - Int(self.avail_out) + self.pointee.avail_out = UInt32(typedOutputPtr.count) + self.pointee.next_out = typedOutputPtr.baseAddress! + rc = cnioextras_z_deflate(self, flag) + return typedOutputPtr.count - Int(self.pointee.avail_out) } return rc diff --git a/Sources/NIOHTTPCompression/HTTPDecompression.swift b/Sources/NIOHTTPCompression/HTTPDecompression.swift index 6732922..9d167d7 100644 --- a/Sources/NIOHTTPCompression/HTTPDecompression.swift +++ b/Sources/NIOHTTPCompression/HTTPDecompression.swift @@ -96,7 +96,7 @@ public enum NIOHTTPDecompression: Sendable { } } - struct Decompressor { + final class Decompressor { /// `15` is the base two logarithm of the maximum window size (the size of the history buffer). It should be in the range 8..15 for this version of the library. /// `32` enables automatic detection of gzip or zlib compression format with automatic header detection. /// @@ -130,14 +130,31 @@ public enum NIOHTTPDecompression: Sendable { static let windowBitsWithAutomaticCompressionFormatDetection: Int32 = 15 + 32 private let limit: NIOHTTPDecompression.DecompressionLimit - private var stream = cnioextras_z_stream() + // zlib stores the address of this stream in its internal state. Keep it in separately + // allocated storage so that its address remains stable for the entire decompression lifecycle. + private let stream: UnsafeMutablePointer private var inflated = 0 + private var isInitialized = false + + // Internal for lifecycle tests: zlib clears this pointer only after inflateEnd releases + // the state it allocated for the stream. + var zlibStateIsAllocated: Bool { + self.stream.pointee.state != nil + } init(limit: NIOHTTPDecompression.DecompressionLimit) { self.limit = limit + self.stream = .allocate(capacity: 1) + self.stream.initialize(to: cnioextras_z_stream()) } - mutating func decompress( + deinit { + self.deinitializeDecoder() + self.stream.deinitialize(count: 1) + self.stream.deallocate() + } + + func decompress( part: inout ByteBuffer, buffer: inout ByteBuffer, compressedLength: Int @@ -152,60 +169,71 @@ public enum NIOHTTPDecompression: Sendable { return result } - mutating func initializeDecoder() throws { - self.stream.zalloc = nil - self.stream.zfree = nil - self.stream.opaque = nil + func initializeDecoder() throws { + try self.initializeDecoder(windowBits: Self.windowBitsWithAutomaticCompressionFormatDetection) + } + + func initializeDecoder(windowBits: Int32) throws { + precondition(!self.isInitialized) + self.stream.pointee.zalloc = nil + self.stream.pointee.zfree = nil + self.stream.pointee.opaque = nil self.inflated = 0 - let rc = CNIOExtrasZlib_inflateInit2(&self.stream, Self.windowBitsWithAutomaticCompressionFormatDetection) + let rc = CNIOExtrasZlib_inflateInit2(self.stream, windowBits) guard rc == CNIOEXTRAS_Z_OK else { throw NIOHTTPDecompression.DecompressionError.initializationError(Int(rc)) } + self.isInitialized = true } - mutating func deinitializeDecoder() { - cnioextras_z_inflateEnd(&self.stream) + @discardableResult + func deinitializeDecoder() -> Int32? { + if self.isInitialized { + self.isInitialized = false + return cnioextras_z_inflateEnd(self.stream) + } + return nil } } } -extension cnioextras_z_stream { - mutating func inflatePart(input: inout ByteBuffer, output: inout ByteBuffer) throws -> InflateResult { +extension UnsafeMutablePointer where Pointee == cnioextras_z_stream { + func inflatePart(input: inout ByteBuffer, output: inout ByteBuffer) throws -> InflateResult { let minimumCapacity = input.readableBytes * 2 var inflateResult = InflateResult(written: 0, complete: false) try input.readWithUnsafeMutableReadableBytes { pointer in - self.avail_in = UInt32(pointer.count) - self.next_in = CNIOExtrasZlib_voidPtr_to_BytefPtr(pointer.baseAddress!) + self.pointee.avail_in = UInt32(pointer.count) + self.pointee.next_in = CNIOExtrasZlib_voidPtr_to_BytefPtr(pointer.baseAddress!) defer { - self.avail_in = 0 - self.next_in = nil - self.avail_out = 0 - self.next_out = nil + self.pointee.avail_in = 0 + self.pointee.next_in = nil + self.pointee.avail_out = 0 + self.pointee.next_out = nil } inflateResult = try self.inflatePart(to: &output, minimumCapacity: minimumCapacity) - return pointer.count - Int(self.avail_in) + return pointer.count - Int(self.pointee.avail_in) } return inflateResult } - private mutating func inflatePart(to buffer: inout ByteBuffer, minimumCapacity: Int) throws -> InflateResult { + private func inflatePart(to buffer: inout ByteBuffer, minimumCapacity: Int) throws -> InflateResult { var rc = CNIOEXTRAS_Z_OK let written = try buffer.writeWithUnsafeMutableBytes(minimumWritableBytes: minimumCapacity) { pointer in - self.avail_out = UInt32(pointer.count) - self.next_out = CNIOExtrasZlib_voidPtr_to_BytefPtr(pointer.baseAddress!) + self.pointee.avail_out = UInt32(pointer.count) + self.pointee.next_out = CNIOExtrasZlib_voidPtr_to_BytefPtr(pointer.baseAddress!) - rc = cnioextras_z_inflate(&self, CNIOEXTRAS_Z_NO_FLUSH) + rc = cnioextras_z_inflate(self, CNIOEXTRAS_Z_NO_FLUSH) guard rc == CNIOEXTRAS_Z_OK || rc == CNIOEXTRAS_Z_STREAM_END else { throw NIOHTTPDecompression.DecompressionError.inflationError(Int(rc)) } - return pointer.count - Int(self.avail_out) + return pointer.count - Int(self.pointee.avail_out) } return InflateResult(written: written, complete: rc == CNIOEXTRAS_Z_STREAM_END) diff --git a/Sources/NIOHTTPCompression/HTTPDecompressor.swift b/Sources/NIOHTTPCompression/HTTPDecompressor.swift index dcbb500..229c9d3 100644 --- a/Sources/NIOHTTPCompression/HTTPDecompressor.swift +++ b/Sources/NIOHTTPCompression/HTTPDecompressor.swift @@ -98,7 +98,7 @@ private final class HTTPDecompressionDecoder: NIOSingleStepByteToMessageDecoder typealias InboundOut = ByteBuffer private let allocator: ByteBufferAllocator - private var decompressor: NIOHTTPDecompression.Decompressor + private let decompressor: NIOHTTPDecompression.Decompressor /// The number of already consumed compressed bytes private var compressedLength: Int private var complete: Bool diff --git a/Sources/NIOHTTPCompression/HTTPRequestCompressor.swift b/Sources/NIOHTTPCompression/HTTPRequestCompressor.swift index c32715c..e27dfe0 100644 --- a/Sources/NIOHTTPCompression/HTTPRequestCompressor.swift +++ b/Sources/NIOHTTPCompression/HTTPRequestCompressor.swift @@ -51,7 +51,7 @@ public final class NIOHTTPRequestCompressor: ChannelOutboundHandler, RemovableCh /// handler state var state: State /// compression handler - var compressor: NIOCompression.Compressor + let compressor: NIOCompression.Compressor /// pending write promise var pendingWritePromise: EventLoopPromise! diff --git a/Sources/NIOHTTPCompression/HTTPResponseCompressor.swift b/Sources/NIOHTTPCompression/HTTPResponseCompressor.swift index 25242b4..c6c6f12 100644 --- a/Sources/NIOHTTPCompression/HTTPResponseCompressor.swift +++ b/Sources/NIOHTTPCompression/HTTPResponseCompressor.swift @@ -126,7 +126,7 @@ public final class HTTPResponseCompressor: ChannelDuplexHandler, RemovableChanne case noDataToWrite } - private var compressor: NIOCompression.Compressor + private let compressor: NIOCompression.Compressor // A queue of accept headers. private var acceptQueue = CircularBuffer<[Substring]>(initialCapacity: 8) @@ -270,7 +270,7 @@ public final class HTTPResponseCompressor: ChannelDuplexHandler, RemovableChanne /// /// Called either when a HTTP end message is received or our flush() method is called. private func emitPendingWrites(context: ChannelHandlerContext) { - let writesToEmit = pendingResponse.flush(compressor: &compressor, allocator: context.channel.allocator) + let writesToEmit = pendingResponse.flush(compressor: compressor, allocator: context.channel.allocator) var pendingPromise = pendingWritePromise if let writeHead = writesToEmit.0 { @@ -372,7 +372,7 @@ private struct PartialHTTPResponse { /// Calling this function resets the buffer, freeing any excess memory allocated in the internal /// buffer and losing all copies of the other HTTP data. At this point it may freely be reused. mutating func flush( - compressor: inout NIOCompression.Compressor, + compressor: NIOCompression.Compressor, allocator: ByteBufferAllocator ) -> (HTTPResponseHead?, ByteBuffer?, HTTPServerResponsePart?) { var outputBody: ByteBuffer? = nil diff --git a/Tests/NIOHTTPCompressionTests/ZlibStreamLifecycleTest.swift b/Tests/NIOHTTPCompressionTests/ZlibStreamLifecycleTest.swift new file mode 100644 index 0000000..7972347 --- /dev/null +++ b/Tests/NIOHTTPCompressionTests/ZlibStreamLifecycleTest.swift @@ -0,0 +1,88 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2026 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import CNIOExtrasZlib +import NIOCore +import XCTest + +@testable import NIOHTTPCompression + +final class ZlibStreamLifecycleTest: XCTestCase { + func testDecompressorCanInitializeAfterInitializationFailure() throws { + let decompressor = NIOHTTPDecompression.Decompressor(limit: .none) + + XCTAssertThrowsError(try decompressor.initializeDecoder(windowBits: 100)) { error in + XCTAssertEqual( + error as? NIOHTTPDecompression.DecompressionError, + .initializationError(Int(CNIOEXTRAS_Z_STREAM_ERROR)) + ) + } + + XCTAssertNoThrow(try decompressor.initializeDecoder()) + XCTAssertEqual(decompressor.deinitializeDecoder(), CNIOEXTRAS_Z_OK) + // Explicit deinitialization is idempotent so the stream owner cannot call inflateEnd twice. + XCTAssertNil(decompressor.deinitializeDecoder()) + } + + func testStreamOwnersSupportMultipleCompressionLifecycles() throws { + let allocator = ByteBufferAllocator() + let compressor = NIOCompression.Compressor() + let decompressor = NIOHTTPDecompression.Decompressor(limit: .none) + + for (encoding, chunks) in [ + (NIOCompression.Algorithm.gzip, ["first compression ", "lifecycle"]), + (NIOCompression.Algorithm.deflate, ["second compression ", "lifecycle"]), + ] { + compressor.initialize(encoding: encoding) + XCTAssertTrue(compressor.zlibStateIsAllocated) + var firstInput = allocator.buffer(capacity: chunks[0].utf8.count) + firstInput.writeString(chunks[0]) + var compressed = compressor.compress(inputBuffer: &firstInput, allocator: allocator, finalise: false) + + var secondInput = allocator.buffer(capacity: chunks[1].utf8.count) + secondInput.writeString(chunks[1]) + var finalCompressed = compressor.compress(inputBuffer: &secondInput, allocator: allocator, finalise: true) + compressed.writeBuffer(&finalCompressed) + XCTAssertEqual(compressor.shutdown(), CNIOEXTRAS_Z_OK) + XCTAssertNil(compressor.shutdownIfActive()) + XCTAssertFalse(compressor.zlibStateIsAllocated) + + let text = chunks.joined() + let compressedLength = compressed.readableBytes + var firstCompressedPart = compressed.readSlice(length: compressedLength / 2)! + var output = allocator.buffer(capacity: text.utf8.count) + try decompressor.initializeDecoder() + XCTAssertTrue(decompressor.zlibStateIsAllocated) + let firstResult = try decompressor.decompress( + part: &firstCompressedPart, + buffer: &output, + compressedLength: compressedLength + ) + let finalResult = try decompressor.decompress( + part: &compressed, + buffer: &output, + compressedLength: compressedLength + ) + XCTAssertEqual(decompressor.deinitializeDecoder(), CNIOEXTRAS_Z_OK) + XCTAssertFalse(decompressor.zlibStateIsAllocated) + + XCTAssertFalse(firstResult.complete) + XCTAssertTrue(finalResult.complete) + XCTAssertEqual(firstCompressedPart.readableBytes, 0) + XCTAssertEqual(compressed.readableBytes, 0) + XCTAssertEqual(output.readString(length: output.readableBytes), text) + } + } + +}