mirror of
https://github.com/apple/swift-nio
synced 2026-11-22 04:30:41 +00:00
Adopt package-benchmark (#2534)
# Motivation We want to migrate our allocation and later on also our performance tests to use the `package-benchmark` plugin. This plugin makes writing benchmarks way easier than our current setup. Furthermore, debugging benchmarks is also possible from within Xcode now. # Modification This PR adds the setup for the benchmarking infrastructure and connects it with out # Result Allocations tests are more accessible and easier to iterate.
This commit is contained in:
1 parent
4ab8b98f9d
commit
d782de84df
18 files changed
+278
-4
No files matched your search
@@ -0,0 +1,9 @@
|
||||
.DS_Store
|
||||
/.build
|
||||
/Packages
|
||||
xcuserdata/
|
||||
DerivedData/
|
||||
.swiftpm/configuration/registries.json
|
||||
.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
|
||||
.netrc
|
||||
.benchmarkBaselines/
|
||||
@@ -0,0 +1,32 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the SwiftNIO open source project
|
||||
//
|
||||
// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
import Benchmark
|
||||
|
||||
let benchmarks = {
|
||||
let defaultMetrics: [BenchmarkMetric] = [
|
||||
.mallocCountTotal,
|
||||
]
|
||||
|
||||
Benchmark(
|
||||
"TCPEcho",
|
||||
configuration: .init(
|
||||
metrics: defaultMetrics,
|
||||
timeUnits: .milliseconds,
|
||||
scalingFactor: .mega
|
||||
)
|
||||
) { benchmark in
|
||||
try runTCPEcho(numberOfWrites: benchmark.scaledIterations.upperBound)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the SwiftNIO open source project
|
||||
//
|
||||
// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
import NIOCore
|
||||
import NIOPosix
|
||||
|
||||
private final class EchoChannelHandler: ChannelInboundHandler {
|
||||
fileprivate typealias InboundIn = ByteBuffer
|
||||
|
||||
func channelRead(context: ChannelHandlerContext, data: NIOAny) {
|
||||
context.writeAndFlush(data, promise: nil)
|
||||
}
|
||||
}
|
||||
|
||||
private final class EchoRequestChannelHandler: ChannelInboundHandler {
|
||||
fileprivate typealias InboundIn = ByteBuffer
|
||||
|
||||
private let bufferSize = 10000
|
||||
private let numberOfWrites: Int
|
||||
private var batchCount = 0
|
||||
private let data: NIOAny
|
||||
private let readsCompletePromise: EventLoopPromise<Void>
|
||||
private var receivedData = 0
|
||||
|
||||
init(numberOfWrites: Int, readsCompletePromise: EventLoopPromise<Void>) {
|
||||
self.numberOfWrites = numberOfWrites
|
||||
self.readsCompletePromise = readsCompletePromise
|
||||
self.data = NIOAny(ByteBuffer(repeating: 0, count: self.bufferSize))
|
||||
}
|
||||
|
||||
func channelActive(context: ChannelHandlerContext) {
|
||||
for _ in 0..<self.numberOfWrites {
|
||||
context.writeAndFlush(data, promise: nil)
|
||||
}
|
||||
}
|
||||
|
||||
func channelRead(context: ChannelHandlerContext, data: NIOAny) {
|
||||
let buffer = self.unwrapInboundIn(data)
|
||||
self.receivedData += buffer.readableBytes
|
||||
|
||||
if self.receivedData == self.numberOfWrites * self.bufferSize {
|
||||
self.readsCompletePromise.succeed()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runTCPEcho(numberOfWrites: Int) throws {
|
||||
let eventLoop = MultiThreadedEventLoopGroup.singleton.next()
|
||||
let serverChannel = try ServerBootstrap(group: eventLoop)
|
||||
.childChannelInitializer { channel in
|
||||
channel.eventLoop.makeCompletedFuture {
|
||||
try channel.pipeline.syncOperations.addHandler(EchoChannelHandler())
|
||||
}
|
||||
}
|
||||
.bind(
|
||||
host: "127.0.0.1",
|
||||
port: 0
|
||||
).wait()
|
||||
|
||||
let readsCompletePromise = eventLoop.makePromise(of: Void.self)
|
||||
let clientChannel = try ClientBootstrap(group: eventLoop)
|
||||
.channelInitializer { channel in
|
||||
channel.eventLoop.makeCompletedFuture {
|
||||
let echoRequestHandler = EchoRequestChannelHandler(
|
||||
numberOfWrites: numberOfWrites,
|
||||
readsCompletePromise: readsCompletePromise
|
||||
)
|
||||
try channel.pipeline.syncOperations.addHandler(echoRequestHandler)
|
||||
}
|
||||
}
|
||||
.connect(
|
||||
host: "127.0.0.1",
|
||||
port: serverChannel.localAddress!.port!
|
||||
).wait()
|
||||
|
||||
// Waiting for the client to collect all echoed data.
|
||||
try readsCompletePromise.futureResult.wait()
|
||||
try serverChannel.close().wait()
|
||||
try clientChannel.close().wait()
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
// swift-tools-version: 5.7
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the SwiftCertificates open source project
|
||||
//
|
||||
// Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
// See CONTRIBUTORS.txt for the list of SwiftCertificates project authors
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
import PackageDescription
|
||||
|
||||
let package = Package(
|
||||
name: "benchmarks",
|
||||
platforms: [
|
||||
.macOS(.v13),
|
||||
],
|
||||
dependencies: [
|
||||
.package(path: "../"),
|
||||
.package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.11.1"),
|
||||
],
|
||||
targets: [
|
||||
.executableTarget(
|
||||
name: "NIOPosixBenchmarks",
|
||||
dependencies: [
|
||||
.product(name: "Benchmark", package: "package-benchmark"),
|
||||
.product(name: "NIOCore", package: "swift-nio"),
|
||||
.product(name: "NIOPosix", package: "swift-nio"),
|
||||
],
|
||||
path: "Benchmarks/NIOPosixBenchmarks",
|
||||
plugins: [
|
||||
.plugin(name: "BenchmarkPlugin", package: "package-benchmark")
|
||||
]
|
||||
),
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"mallocCountTotal" : 93
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"mallocCountTotal" : 95
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"mallocCountTotal" : 95
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"mallocCountTotal" : 95
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"mallocCountTotal" : 93
|
||||
}
|
||||
@@ -352,6 +352,19 @@ apt-get install -y git curl libatomic1 libxml2 netcat-openbsd lsof perl
|
||||
dnf install swift-lang /usr/bin/nc /usr/bin/lsof /usr/bin/shasum
|
||||
```
|
||||
|
||||
### Benchmarks
|
||||
|
||||
Benchmarks for `swift-nio` are in a separate Swift Package in the `Benchmarks` subfolder of this repository.
|
||||
They use the [`package-benchmark`](https://github.com/ordo-one/package-benchmark) plugin.
|
||||
Benchmarks depends on the [`jemalloc`](https://jemalloc.net) memory allocation library, which is used by `package-benchmark` to capture memory allocation statistics.
|
||||
An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`.
|
||||
Afterwards you can run the benchmarks from CLI by going to the `Benchmarks` subfolder (e.g. `cd Benchmarks`) and invoking:
|
||||
```
|
||||
swift package benchmark
|
||||
```
|
||||
|
||||
For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark).
|
||||
|
||||
[ch]: https://swiftpackageindex.com/apple/swift-nio/main/documentation/niocore/channelhandler
|
||||
[c]: https://swiftpackageindex.com/apple/swift-nio/main/documentation/niocore/channel
|
||||
[chc]: https://swiftpackageindex.com/apple/swift-nio/main/documentation/niocore/channelhandlercontext
|
||||
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
##===----------------------------------------------------------------------===##
|
||||
##
|
||||
## This source file is part of the SwiftNIO open source project
|
||||
##
|
||||
## Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors
|
||||
## Licensed under Apache License v2.0
|
||||
##
|
||||
## See LICENSE.txt for license information
|
||||
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
|
||||
##
|
||||
## SPDX-License-Identifier: Apache-2.0
|
||||
##
|
||||
##===----------------------------------------------------------------------===##
|
||||
##===----------------------------------------------------------------------===##
|
||||
##
|
||||
## This source file is part of the SwiftCertificates open source project
|
||||
##
|
||||
## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors
|
||||
## Licensed under Apache License v2.0
|
||||
##
|
||||
## See LICENSE.txt for license information
|
||||
## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors
|
||||
##
|
||||
## SPDX-License-Identifier: Apache-2.0
|
||||
##
|
||||
##===----------------------------------------------------------------------===##
|
||||
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
target_repo=${2-"$here/.."}
|
||||
|
||||
for f in 57 58 59 510 -nightly; do
|
||||
echo "swift$f"
|
||||
|
||||
docker_file=$(if [[ "$f" == "-nightly" ]]; then f=main; fi && ls "$target_repo/docker/docker-compose."*"$f"*".yaml")
|
||||
|
||||
docker-compose -f docker/docker-compose.yaml -f $docker_file run update-benchmark-baseline
|
||||
done
|
||||
+4
-1
@@ -1,5 +1,5 @@
|
||||
ARG swift_version=5.7
|
||||
ARG ubuntu_version=bionic
|
||||
ARG ubuntu_version=jammy
|
||||
ARG base_image=swift:$swift_version-$ubuntu_version
|
||||
FROM $base_image
|
||||
# needed to do again after FROM due to docker limitation
|
||||
@@ -19,6 +19,9 @@ RUN apt-get update && apt-get install -y lsof dnsutils netcat-openbsd net-tools
|
||||
# ruby
|
||||
RUN apt-get update && apt-get install -y ruby ruby-dev libsqlite3-dev build-essential
|
||||
|
||||
# install jemalloc for running allocation benchmarks
|
||||
RUN apt-get update & apt-get install -y libjemalloc-dev
|
||||
|
||||
# tools
|
||||
RUN mkdir -p $HOME/.tools
|
||||
RUN echo 'export PATH="$HOME/.tools:$PATH"' >> $HOME/.profile
|
||||
@@ -76,6 +76,11 @@ services:
|
||||
performance-test:
|
||||
image: swift-nio:22.04-5.10
|
||||
|
||||
update-benchmark-baseline:
|
||||
image: swift-nio:22.04-5.10
|
||||
environment:
|
||||
- SWIFT_VERSION=5.10
|
||||
|
||||
shell:
|
||||
image: swift-nio:22.04-5.10
|
||||
|
||||
|
||||
@@ -76,6 +76,11 @@ services:
|
||||
performance-test:
|
||||
image: swift-nio:22.04-5.7
|
||||
|
||||
update-benchmark-baseline:
|
||||
image: swift-nio:22.04-5.7
|
||||
environment:
|
||||
- SWIFT_VERSION=5.7
|
||||
|
||||
shell:
|
||||
image: swift-nio:22.04-5.7
|
||||
|
||||
|
||||
@@ -77,6 +77,11 @@ services:
|
||||
performance-test:
|
||||
image: swift-nio:22.04-5.8
|
||||
|
||||
update-benchmark-baseline:
|
||||
image: swift-nio:22.04-5.8
|
||||
environment:
|
||||
- SWIFT_VERSION=5.8
|
||||
|
||||
shell:
|
||||
image: swift-nio:22.04-5.8
|
||||
|
||||
|
||||
@@ -77,6 +77,11 @@ services:
|
||||
performance-test:
|
||||
image: swift-nio:22.04-5.9
|
||||
|
||||
update-benchmark-baseline:
|
||||
image: swift-nio:22.04-5.9
|
||||
environment:
|
||||
- SWIFT_VERSION=5.9
|
||||
|
||||
shell:
|
||||
image: swift-nio:22.04-5.9
|
||||
|
||||
|
||||
@@ -76,6 +76,11 @@ services:
|
||||
performance-test:
|
||||
image: swift-nio:22.04-main
|
||||
|
||||
update-benchmark-baseline:
|
||||
image: swift-nio:22.04-main
|
||||
environment:
|
||||
- SWIFT_VERSION=main
|
||||
|
||||
shell:
|
||||
image: swift-nio:22.04-main
|
||||
|
||||
|
||||
@@ -16,8 +16,8 @@ services:
|
||||
depends_on: [runtime-setup]
|
||||
volumes:
|
||||
- ~/.ssh:/root/.ssh
|
||||
- ..:/code:z
|
||||
working_dir: /code
|
||||
- ..:/swift-nio:z
|
||||
working_dir: /swift-nio
|
||||
cap_drop:
|
||||
- CAP_NET_RAW
|
||||
- CAP_NET_BIND_SERVICE
|
||||
@@ -40,12 +40,16 @@ services:
|
||||
|
||||
test:
|
||||
<<: *common
|
||||
command: /bin/bash -xcl "uname -a && swift -version && swift $${SWIFT_TEST_VERB-test} $${FORCE_TEST_DISCOVERY-} $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-} && ./scripts/integration_tests.sh $${INTEGRATION_TESTS_ARG-}"
|
||||
command: /bin/bash -xcl "uname -a && swift -version && swift $${SWIFT_TEST_VERB-test} $${FORCE_TEST_DISCOVERY-} $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-} && ./scripts/integration_tests.sh $${INTEGRATION_TESTS_ARG-} && cd Benchmarks && swift package benchmark baseline check --check-absolute-path Thresholds/$${SWIFT_VERSION-}/"
|
||||
|
||||
performance-test:
|
||||
<<: *common
|
||||
command: /bin/bash -xcl "swift build -c release -Xswiftc -Xllvm -Xswiftc -align-all-functions=5 -Xswiftc -Xllvm -Xswiftc -align-all-blocks=5 && ./.build/release/NIOPerformanceTester"
|
||||
|
||||
update-benchmark-baseline:
|
||||
<<: *common
|
||||
command: /bin/bash -xcl "cd Benchmarks && swift package --disable-sandbox --scratch-path .build/$${SWIFT_VERSION-}/ --allow-writing-to-package-directory benchmark --format metricP90AbsoluteThresholds --path Thresholds/$${SWIFT_VERSION-}/"
|
||||
|
||||
# util
|
||||
|
||||
shell:
|
||||
|
||||
Reference in new issue
Block a user