diff --git a/Benchmarks/Benchmarks/NIOPosixBenchmarks/Benchmarks.swift b/Benchmarks/Benchmarks/NIOPosixBenchmarks/Benchmarks.swift index 947934644..db30879fe 100644 --- a/Benchmarks/Benchmarks/NIOPosixBenchmarks/Benchmarks.swift +++ b/Benchmarks/Benchmarks/NIOPosixBenchmarks/Benchmarks.swift @@ -171,4 +171,38 @@ let benchmarks = { await actor.foo() } } + + // MARK: - NIOThreadPool submit benchmarks + + // Serial wakeup: submit one item, wait for completion, repeat. + // Every submit hits N sleeping threads — this is where wake-one + // vs wake-all (thundering herd) matters most. + let pool16 = NIOThreadPool(numberOfThreads: 16) + let pool4 = NIOThreadPool(numberOfThreads: 4) + + Benchmark( + "NIOThreadPool.serial_wakeup(16 threads)", + configuration: .init( + metrics: [.wallClock, .cpuUser, .cpuSystem, .cpuTotal, .contextSwitches, .syscalls], + maxDuration: .seconds(30), + maxIterations: 30, + setup: { pool16.start() }, + teardown: { try! pool16.syncShutdownGracefully() } + ) + ) { _ in + runNIOThreadPoolSerialWakeup(pool: pool16, count: 10_000) + } + + Benchmark( + "NIOThreadPool.serial_wakeup(4 threads)", + configuration: .init( + metrics: [.wallClock, .cpuUser, .cpuSystem, .cpuTotal, .contextSwitches, .syscalls], + maxDuration: .seconds(30), + maxIterations: 30, + setup: { pool4.start() }, + teardown: { try! pool4.syncShutdownGracefully() } + ) + ) { _ in + runNIOThreadPoolSerialWakeup(pool: pool4, count: 10_000) + } } diff --git a/Benchmarks/Benchmarks/NIOPosixBenchmarks/NIOThreadPoolBenchmark.swift b/Benchmarks/Benchmarks/NIOPosixBenchmarks/NIOThreadPoolBenchmark.swift new file mode 100644 index 000000000..b5c9361f9 --- /dev/null +++ b/Benchmarks/Benchmarks/NIOPosixBenchmarks/NIOThreadPoolBenchmark.swift @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2026 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Dispatch +import NIOCore +import NIOPosix + +func runNIOThreadPoolSerialWakeup(pool: NIOThreadPool, count: Int) { + let sem = DispatchSemaphore(value: 0) + for _ in 0.. - Kim de Vos - Konrad `ktoso` Malawski +- Kushal Pisavadia - Kyle Bashour - Kyle Browning - Kyle Jessup diff --git a/Sources/NIOConcurrencyHelpers/NIOThreadPoolWorkAvailable.swift b/Sources/NIOConcurrencyHelpers/NIOThreadPoolWorkAvailable.swift new file mode 100644 index 000000000..04e332551 --- /dev/null +++ b/Sources/NIOConcurrencyHelpers/NIOThreadPoolWorkAvailable.swift @@ -0,0 +1,187 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2026 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(Darwin) +import Darwin +#elseif os(Windows) +import ucrt +import WinSDK +#elseif canImport(Glibc) +@preconcurrency import Glibc +#elseif canImport(Musl) +@preconcurrency import Musl +#elseif canImport(Bionic) +@preconcurrency import Bionic +#elseif canImport(WASILibc) +@preconcurrency import WASILibc +#if canImport(wasi_pthread) +import wasi_pthread +#endif +#else +#error("The NIOThreadPoolWorkAvailable module was unable to identify your C library.") +#endif + +/// A specialized synchronization primitive for ``NIOThreadPool`` that uses +/// `pthread_cond_signal` (wake-one) instead of `pthread_cond_broadcast` (wake-all) +/// when work is enqueued, eliminating the thundering-herd problem. +/// +/// This type manages a `workAvailable` counter under a mutex, paired with a +/// condition variable. Threads waiting for work block until `workAvailable > 0`. +@usableFromInline +package struct NIOThreadPoolWorkAvailable: @unchecked Sendable { + @usableFromInline + final class _Storage { + @usableFromInline + let lock: NIOLock + + @usableFromInline + var workAvailable: Int + + #if os(Windows) + @usableFromInline + let cond: UnsafeMutablePointer = + UnsafeMutablePointer.allocate(capacity: 1) + #elseif os(OpenBSD) + @usableFromInline + let cond: UnsafeMutablePointer = + UnsafeMutablePointer.allocate(capacity: 1) + #elseif (compiler(<6.1) && !os(WASI)) || (compiler(>=6.1) && _runtime(_multithreaded)) + @usableFromInline + let cond: UnsafeMutablePointer = + UnsafeMutablePointer.allocate(capacity: 1) + #endif + + @usableFromInline + init() { + self.lock = NIOLock() + self.workAvailable = 0 + #if os(Windows) + InitializeConditionVariable(self.cond) + #elseif (compiler(<6.1) && !os(WASI)) || (compiler(>=6.1) && _runtime(_multithreaded)) + let err = pthread_cond_init(self.cond, nil) + precondition(err == 0, "\(#function) failed in pthread_cond_init with error \(err)") + #endif + } + + deinit { + #if os(Windows) + // condition variables do not need to be explicitly destroyed + self.cond.deallocate() + #elseif (compiler(<6.1) && !os(WASI)) || (compiler(>=6.1) && _runtime(_multithreaded)) + let err = pthread_cond_destroy(self.cond) + precondition(err == 0, "\(#function) failed in pthread_cond_destroy with error \(err)") + self.cond.deallocate() + #endif + } + } + + @usableFromInline + let _storage: _Storage + + @usableFromInline + package enum Signal: Sendable { + /// No signal after unlock. + case none + /// Wake one waiting thread (``pthread_cond_signal``). + case signalOne + /// Wake all waiting threads (``pthread_cond_broadcast``). + case broadcastAll + } + + @inlinable + package init() { + self._storage = _Storage() + } + + /// Lock, run `body`, apply the returned delta to `workAvailable`, + /// unlock, then signal as indicated by the return value. + @inlinable + package func withLock( + _ body: () -> (workDelta: Int, signal: Signal, result: Result) + ) -> Result { + self._storage.lock.lock() + let (workDelta, signal, result) = body() + self._storage.workAvailable += workDelta + self._storage.lock.unlock() + self._signal(signal) + return result + } + + /// Lock, run `body`, set `workAvailable` to the returned value if non-nil, + /// unlock, then signal as indicated. Use for shutdown where an + /// absolute value is needed rather than a delta. + @inlinable + package func withLockSettingWorkAvailable( + _ body: () -> (workAvailable: Int?, signal: Signal, result: Result) + ) -> Result { + self._storage.lock.lock() + let (workAvailable, signal, result) = body() + if let workAvailable = workAvailable { + self._storage.workAvailable = workAvailable + } + self._storage.lock.unlock() + self._signal(signal) + return result + } + + /// Lock, wait while `workAvailable <= 0`, run `body`, apply the + /// returned delta to `workAvailable`, unlock, then signal as indicated. + @inlinable + package func withLockWaitingForWork( + _ body: () -> (workDelta: Int, signal: Signal, result: Result) + ) -> Result { + self._storage.lock.lock() + while self._storage.workAvailable <= 0 { + self._storage.lock.withLockPrimitive { mutex in + #if os(Windows) + let ok = SleepConditionVariableSRW(self._storage.cond, mutex, INFINITE, 0) + precondition( + ok, + "\(#function) failed in SleepConditionVariableSRW with error \(GetLastError())" + ) + #elseif (compiler(<6.1) && !os(WASI)) || (compiler(>=6.1) && _runtime(_multithreaded)) + let err = pthread_cond_wait(self._storage.cond, mutex) + precondition(err == 0, "\(#function) failed in pthread_cond_wait with error \(err)") + #endif + } + } + let (workDelta, signal, result) = body() + self._storage.workAvailable += workDelta + self._storage.lock.unlock() + self._signal(signal) + return result + } + + @inlinable + func _signal(_ signal: Signal) { + switch signal { + case .none: + break + case .signalOne: + #if os(Windows) + WakeConditionVariable(self._storage.cond) + #elseif (compiler(<6.1) && !os(WASI)) || (compiler(>=6.1) && _runtime(_multithreaded)) + let err = pthread_cond_signal(self._storage.cond) + precondition(err == 0, "\(#function) failed in pthread_cond_signal with error \(err)") + #endif + case .broadcastAll: + #if os(Windows) + WakeAllConditionVariable(self._storage.cond) + #elseif (compiler(<6.1) && !os(WASI)) || (compiler(>=6.1) && _runtime(_multithreaded)) + let err = pthread_cond_broadcast(self._storage.cond) + precondition(err == 0, "\(#function) failed in pthread_cond_broadcast with error \(err)") + #endif + } + } +} diff --git a/Sources/NIOPerformanceTester/NIOThreadPoolSubmitBenchmark.swift b/Sources/NIOPerformanceTester/NIOThreadPoolSubmitBenchmark.swift new file mode 100644 index 000000000..5ebd702ba --- /dev/null +++ b/Sources/NIOPerformanceTester/NIOThreadPoolSubmitBenchmark.swift @@ -0,0 +1,51 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2026 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Dispatch +import NIOCore +import NIOPosix + +/// Submit one item at a time, waiting for completion before the next. +/// Every submit hits N sleeping threads — isolates wake-one vs wake-all. +final class NIOThreadPoolSerialWakeupBenchmark: Benchmark { + private let numberOfThreads: Int + private let numberOfTasks: Int + private var pool: NIOThreadPool! + + init(numberOfThreads: Int, numberOfTasks: Int) { + self.numberOfThreads = numberOfThreads + self.numberOfTasks = numberOfTasks + } + + func setUp() throws { + self.pool = NIOThreadPool(numberOfThreads: self.numberOfThreads) + self.pool.start() + } + + func tearDown() { + try! self.pool.syncShutdownGracefully() + } + + func run() throws -> Int { + let sem = DispatchSemaphore(value: 0) + for _ in 0.. - private var threads: [NIOThread]? = nil // protected by `conditionLock` + internal let _workAvailable: NIOThreadPoolWorkAvailable + private var threads: [NIOThread]? = nil // protected by `_workAvailable` @usableFromInline internal var _state: State = .stopped @@ -148,7 +134,7 @@ public final class NIOThreadPool { return } - let threadsToJoin = self._conditionLock.withLock { + let threadsToJoin = self._workAvailable.withLockSettingWorkAvailable { switch self._state { case .running(let items): self._state = .modifying @@ -162,11 +148,12 @@ public final class NIOThreadPool { let threads = self.threads! self.threads = nil - // Each thread has work to do: shutdown. - return (unlockWith: .hasWork, result: threads) + // Wake all threads so they can see .shuttingDown and exit. + return (workAvailable: self.numberOfThreads, signal: .broadcastAll, result: threads) case .shuttingDown, .stopped: - return (unlockWith: nil, result: []) + // Don't modify workAvailable - threads may still be waking up to see .shuttingDown + return (workAvailable: nil, signal: .none, result: []) case .modifying: fatalError(".modifying state misuse") @@ -195,8 +182,7 @@ public final class NIOThreadPool { } private func _submit(id: Int?, _ body: @escaping WorkItem) { - let submitted = self._conditionLock.withLock { - let workState: _WorkState + let submitted = self._workAvailable.withLock { let submitted: Bool switch self._state { @@ -204,18 +190,16 @@ public final class NIOThreadPool { self._state = .modifying items.append(.init(workItem: body, id: id)) self._state = .running(items) - workState = items.isEmpty ? .hasNoWork : .hasWork submitted = true case .shuttingDown, .stopped: - workState = .hasNoWork submitted = false case .modifying: fatalError(".modifying state misuse") } - return (unlockWith: workState, result: submitted) + return (workDelta: submitted ? 1 : 0, signal: submitted ? .signalOne : .none, result: submitted) } // if item couldn't be added run it immediately indicating that it couldn't be run @@ -244,7 +228,7 @@ public final class NIOThreadPool { private init(numberOfThreads: Int, canBeStopped: Bool) { self.numberOfThreads = numberOfThreads self.canBeStopped = canBeStopped - self._conditionLock = ConditionLock(value: .hasNoWork) + self._workAvailable = NIOThreadPoolWorkAvailable() } // Do not rename or remove this function. @@ -256,9 +240,7 @@ public final class NIOThreadPool { // when only interested in on-CPU work. @inlinable internal func _blockingWaitForWork(identifier: Int) -> (item: WorkItem, state: WorkItemState)? { - self._conditionLock.withLock(when: .hasWork) { - () -> (unlockWith: _WorkState, result: (WorkItem, WorkItemState)?) in - let workState: _WorkState + self._workAvailable.withLockWaitingForWork { let result: (WorkItem, WorkItemState)? switch self._state { @@ -275,7 +257,6 @@ public final class NIOThreadPool { self._state = .running(items) - workState = items.isEmpty ? .hasNoWork : .hasWork result = (itemAndID.workItem, state) case .shuttingDown(var aliveStates): @@ -284,8 +265,6 @@ public final class NIOThreadPool { aliveStates[identifier] = false self._state = .shuttingDown(aliveStates) - // Unlock with '.hasWork' to resume any other threads waiting to shutdown. - workState = .hasWork result = nil case .stopped: @@ -297,7 +276,7 @@ public final class NIOThreadPool { fatalError(".modifying state misuse") } - return (unlockWith: workState, result: result) + return (workDelta: -1, signal: .none, result: result) } } @@ -320,12 +299,12 @@ public final class NIOThreadPool { } public func _start(threadNamePrefix: String) { - let alreadyRunning = self._conditionLock.withLock { + let alreadyRunning = self._workAvailable.withLock { switch self._state { case .running: // Already running, this has no effect on whether there is more work for the // threads to run. - return (unlockWith: nil, result: true) + return (workDelta: 0, signal: .none, result: true) case .shuttingDown: // This should never happen @@ -336,7 +315,7 @@ public final class NIOThreadPool { assert(self.threads == nil) self.threads = [] self.threads!.reserveCapacity(self.numberOfThreads) - return (unlockWith: .hasNoWork, result: false) + return (workDelta: 0, signal: .none, result: false) case .modifying: fatalError(".modifying state misuse") @@ -355,26 +334,10 @@ public final class NIOThreadPool { // We should keep thread names under 16 characters because Linux doesn't allow more. NIOThread.spawnAndRun(name: "\(threadNamePrefix)\(id)") { thread in readyThreads.withLock { - let threadCount = self._conditionLock.withLock { + let threadCount = self._workAvailable.withLock { self.threads!.append(thread) - let workState: _WorkState - - switch self._state { - case .running(let items): - workState = items.isEmpty ? .hasNoWork : .hasWork - case .shuttingDown: - // The thread has work to do: it's shutting down. - workState = .hasWork - case .stopped: - // Unreachable: .stopped always transitions to .running in the function - // and .stopped is never entered again. - fatalError("Invalid state") - case .modifying: - fatalError(".modifying state misuse") - } - let threadCount = self.threads!.count - return (unlockWith: workState, result: threadCount) + return (workDelta: 0, signal: .none, result: threadCount) } return (unlockWith: threadCount, result: ()) @@ -389,8 +352,8 @@ public final class NIOThreadPool { readyThreads.unlock() func threadCount() -> Int { - self._conditionLock.withLock { - (unlockWith: nil, result: self.threads?.count ?? -1) + self._workAvailable.withLock { + (workDelta: 0, signal: .none, result: self.threads?.count ?? -1) } } assert(threadCount() == self.numberOfThreads) @@ -471,9 +434,9 @@ extension NIOThreadPool { } } } onCancel: { - self._conditionLock.withLock { + self._workAvailable.withLock { self._cancelledWorkIDs.insert(workID) - return (unlockWith: nil, result: ()) + return (workDelta: 0, signal: .none, result: ()) } } }