Adopt package-benchmark (#2534)

# Motivation
We want to migrate our allocation and later on also our performance tests to use the `package-benchmark` plugin. This plugin makes writing benchmarks way easier than our current setup. Furthermore, debugging benchmarks is also possible from within Xcode now.

# Modification
This PR adds the setup for the benchmarking infrastructure and connects it with out

# Result
Allocations tests are more accessible and easier to iterate.
This commit is contained in:
Franz Busch authored and GitHub committed 2023-10-08 13:31:22 -07:00
1 parent 4ab8b98f9d
commit d782de84df
18 files changed
+278 -4

No files matched your search

+9
View File
@@ -0,0 +1,9 @@
.DS_Store
/.build
/Packages
xcuserdata/
DerivedData/
.swiftpm/configuration/registries.json
.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
.netrc
.benchmarkBaselines/
@@ -0,0 +1,32 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
import Benchmark
let benchmarks = {
let defaultMetrics: [BenchmarkMetric] = [
.mallocCountTotal,
]
Benchmark(
"TCPEcho",
configuration: .init(
metrics: defaultMetrics,
timeUnits: .milliseconds,
scalingFactor: .mega
)
) { benchmark in
try runTCPEcho(numberOfWrites: benchmark.scaledIterations.upperBound)
}
}
@@ -0,0 +1,91 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
import NIOCore
import NIOPosix
private final class EchoChannelHandler: ChannelInboundHandler {
fileprivate typealias InboundIn = ByteBuffer
func channelRead(context: ChannelHandlerContext, data: NIOAny) {
context.writeAndFlush(data, promise: nil)
}
}
private final class EchoRequestChannelHandler: ChannelInboundHandler {
fileprivate typealias InboundIn = ByteBuffer
private let bufferSize = 10000
private let numberOfWrites: Int
private var batchCount = 0
private let data: NIOAny
private let readsCompletePromise: EventLoopPromise<Void>
private var receivedData = 0
init(numberOfWrites: Int, readsCompletePromise: EventLoopPromise<Void>) {
self.numberOfWrites = numberOfWrites
self.readsCompletePromise = readsCompletePromise
self.data = NIOAny(ByteBuffer(repeating: 0, count: self.bufferSize))
}
func channelActive(context: ChannelHandlerContext) {
for _ in 0..<self.numberOfWrites {
context.writeAndFlush(data, promise: nil)
}
}
func channelRead(context: ChannelHandlerContext, data: NIOAny) {
let buffer = self.unwrapInboundIn(data)
self.receivedData += buffer.readableBytes
if self.receivedData == self.numberOfWrites * self.bufferSize {
self.readsCompletePromise.succeed()
}
}
}
func runTCPEcho(numberOfWrites: Int) throws {
let eventLoop = MultiThreadedEventLoopGroup.singleton.next()
let serverChannel = try ServerBootstrap(group: eventLoop)
.childChannelInitializer { channel in
channel.eventLoop.makeCompletedFuture {
try channel.pipeline.syncOperations.addHandler(EchoChannelHandler())
}
}
.bind(
host: "127.0.0.1",
port: 0
).wait()
let readsCompletePromise = eventLoop.makePromise(of: Void.self)
let clientChannel = try ClientBootstrap(group: eventLoop)
.channelInitializer { channel in
channel.eventLoop.makeCompletedFuture {
let echoRequestHandler = EchoRequestChannelHandler(
numberOfWrites: numberOfWrites,
readsCompletePromise: readsCompletePromise
)
try channel.pipeline.syncOperations.addHandler(echoRequestHandler)
}
}
.connect(
host: "127.0.0.1",
port: serverChannel.localAddress!.port!
).wait()
// Waiting for the client to collect all echoed data.
try readsCompletePromise.futureResult.wait()
try serverChannel.close().wait()
try clientChannel.close().wait()
}
+41
View File
@@ -0,0 +1,41 @@
// swift-tools-version: 5.7
//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftCertificates open source project
//
// Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftCertificates project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
import PackageDescription
let package = Package(
name: "benchmarks",
platforms: [
.macOS(.v13),
],
dependencies: [
.package(path: "../"),
.package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.11.1"),
],
targets: [
.executableTarget(
name: "NIOPosixBenchmarks",
dependencies: [
.product(name: "Benchmark", package: "package-benchmark"),
.product(name: "NIOCore", package: "swift-nio"),
.product(name: "NIOPosix", package: "swift-nio"),
],
path: "Benchmarks/NIOPosixBenchmarks",
plugins: [
.plugin(name: "BenchmarkPlugin", package: "package-benchmark")
]
),
]
)
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 93
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 95
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 95
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 95
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 93
}
+13
View File
@@ -352,6 +352,19 @@ apt-get install -y git curl libatomic1 libxml2 netcat-openbsd lsof perl
dnf install swift-lang /usr/bin/nc /usr/bin/lsof /usr/bin/shasum
```
### Benchmarks
Benchmarks for `swift-nio` are in a separate Swift Package in the `Benchmarks` subfolder of this repository.
They use the [`package-benchmark`](https://github.com/ordo-one/package-benchmark) plugin.
Benchmarks depends on the [`jemalloc`](https://jemalloc.net) memory allocation library, which is used by `package-benchmark` to capture memory allocation statistics.
An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`.
Afterwards you can run the benchmarks from CLI by going to the `Benchmarks` subfolder (e.g. `cd Benchmarks`) and invoking:
```
swift package benchmark
```
For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark).
[ch]: https://swiftpackageindex.com/apple/swift-nio/main/documentation/niocore/channelhandler
[c]: https://swiftpackageindex.com/apple/swift-nio/main/documentation/niocore/channel
[chc]: https://swiftpackageindex.com/apple/swift-nio/main/documentation/niocore/channelhandlercontext
+41
View File
@@ -0,0 +1,41 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftCertificates open source project
##
## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
set -eu
set -o pipefail
here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
target_repo=${2-"$here/.."}
for f in 57 58 59 510 -nightly; do
echo "swift$f"
docker_file=$(if [[ "$f" == "-nightly" ]]; then f=main; fi && ls "$target_repo/docker/docker-compose."*"$f"*".yaml")
docker-compose -f docker/docker-compose.yaml -f $docker_file run update-benchmark-baseline
done
+4 -1
View File
@@ -1,5 +1,5 @@
ARG swift_version=5.7
ARG ubuntu_version=bionic
ARG ubuntu_version=jammy
ARG base_image=swift:$swift_version-$ubuntu_version
FROM $base_image
# needed to do again after FROM due to docker limitation
@@ -19,6 +19,9 @@ RUN apt-get update && apt-get install -y lsof dnsutils netcat-openbsd net-tools
# ruby
RUN apt-get update && apt-get install -y ruby ruby-dev libsqlite3-dev build-essential
# install jemalloc for running allocation benchmarks
RUN apt-get update & apt-get install -y libjemalloc-dev
# tools
RUN mkdir -p $HOME/.tools
RUN echo 'export PATH="$HOME/.tools:$PATH"' >> $HOME/.profile
+5
View File
@@ -76,6 +76,11 @@ services:
performance-test:
image: swift-nio:22.04-5.10
update-benchmark-baseline:
image: swift-nio:22.04-5.10
environment:
- SWIFT_VERSION=5.10
shell:
image: swift-nio:22.04-5.10
+5
View File
@@ -76,6 +76,11 @@ services:
performance-test:
image: swift-nio:22.04-5.7
update-benchmark-baseline:
image: swift-nio:22.04-5.7
environment:
- SWIFT_VERSION=5.7
shell:
image: swift-nio:22.04-5.7
+5
View File
@@ -77,6 +77,11 @@ services:
performance-test:
image: swift-nio:22.04-5.8
update-benchmark-baseline:
image: swift-nio:22.04-5.8
environment:
- SWIFT_VERSION=5.8
shell:
image: swift-nio:22.04-5.8
+5
View File
@@ -77,6 +77,11 @@ services:
performance-test:
image: swift-nio:22.04-5.9
update-benchmark-baseline:
image: swift-nio:22.04-5.9
environment:
- SWIFT_VERSION=5.9
shell:
image: swift-nio:22.04-5.9
+5
View File
@@ -76,6 +76,11 @@ services:
performance-test:
image: swift-nio:22.04-main
update-benchmark-baseline:
image: swift-nio:22.04-main
environment:
- SWIFT_VERSION=main
shell:
image: swift-nio:22.04-main
+7 -3
View File
@@ -16,8 +16,8 @@ services:
depends_on: [runtime-setup]
volumes:
- ~/.ssh:/root/.ssh
- ..:/code:z
working_dir: /code
- ..:/swift-nio:z
working_dir: /swift-nio
cap_drop:
- CAP_NET_RAW
- CAP_NET_BIND_SERVICE
@@ -40,12 +40,16 @@ services:
test:
<<: *common
command: /bin/bash -xcl "uname -a && swift -version && swift $${SWIFT_TEST_VERB-test} $${FORCE_TEST_DISCOVERY-} $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-} && ./scripts/integration_tests.sh $${INTEGRATION_TESTS_ARG-}"
command: /bin/bash -xcl "uname -a && swift -version && swift $${SWIFT_TEST_VERB-test} $${FORCE_TEST_DISCOVERY-} $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-} && ./scripts/integration_tests.sh $${INTEGRATION_TESTS_ARG-} && cd Benchmarks && swift package benchmark baseline check --check-absolute-path Thresholds/$${SWIFT_VERSION-}/"
performance-test:
<<: *common
command: /bin/bash -xcl "swift build -c release -Xswiftc -Xllvm -Xswiftc -align-all-functions=5 -Xswiftc -Xllvm -Xswiftc -align-all-blocks=5 && ./.build/release/NIOPerformanceTester"
update-benchmark-baseline:
<<: *common
command: /bin/bash -xcl "cd Benchmarks && swift package --disable-sandbox --scratch-path .build/$${SWIFT_VERSION-}/ --allow-writing-to-package-directory benchmark --format metricP90AbsoluteThresholds --path Thresholds/$${SWIFT_VERSION-}/"
# util
shell: