Switch allocation benchmarks to GHA (#488)

### Motivation:

This will allow us to decommission the old CI hooks.

### Modifications:

* Migrate the existing tests to use the package-benchmark framework.
* Delete the obsolete integration tests
* Remove allocations test references from the docker files.

### Result:

Allocation counts are checked using GitHub Actions.
This commit is contained in:
Rick Newton-Rogers authored and GitHub committed 2024-10-25 16:17:41 +01:00
1 parent 61cfaf9798
commit 75e5a978e8
26 files changed
+239 -272

No files matched your search

+7 -1
View File
@@ -25,7 +25,13 @@ jobs:
uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
with:
name: "Integration test"
matrix_linux_command: "apt-get update -yq && apt-get install -yq execstack lsof dnsutils netcat-openbsd net-tools expect curl jq && ./scripts/integration_tests.sh -f test_01_renegotiation"
matrix_linux_command: "apt-get update -yq && apt-get install -yq execstack lsof dnsutils netcat-openbsd net-tools expect curl jq && ./scripts/integration_tests.sh"
benchmarks:
name: Benchmarks
uses: apple/swift-nio/.github/workflows/benchmarks.yml@main
with:
benchmark_package_path: "Benchmarks"
cxx-interop:
name: Cxx interop
+6
View File
@@ -22,3 +22,9 @@ jobs:
with:
name: "Integration test"
matrix_linux_command: "apt-get update -yq && apt-get install -yq execstack lsof dnsutils netcat-openbsd net-tools expect curl jq && ./scripts/integration_tests.sh -f test_01_renegotiation"
benchmarks:
name: Benchmarks
uses: apple/swift-nio/.github/workflows/benchmarks.yml@main
with:
benchmark_package_path: "Benchmarks"
@@ -0,0 +1,54 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
import Benchmark
import NIOCore
import NIOEmbedded
import NIOSSL
let benchmarks = {
let defaultMetrics: [BenchmarkMetric] = [
.mallocCountTotal
]
Benchmark(
"SimpleHandshake",
configuration: .init(
metrics: defaultMetrics,
scalingFactor: .kilo,
maxDuration: .seconds(10_000_000),
maxIterations: 10,
thresholds: [.mallocCountTotal: .init(absolute: [.p90: 2000])]
)
) { benchmark in
try runSimpleHandshake(
handshakeCount: benchmark.scaledIterations.upperBound
)
}
Benchmark(
"ManyWrites",
configuration: .init(
metrics: defaultMetrics,
scalingFactor: .kilo,
maxDuration: .seconds(10_000_000),
maxIterations: 10,
thresholds: [.mallocCountTotal: .init(absolute: [.p90: 2000])]
)
) { benchmark in
try runManyWrites(
writeCount: benchmark.scaledIterations.upperBound
)
}
}
@@ -0,0 +1,60 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
import NIOCore
import NIOEmbedded
import NIOSSL
func runManyWrites(writeCount: Int) throws {
let serverContext = try NIOSSLContext(
configuration: .makeServerConfiguration(
certificateChain: [.certificate(.forTesting())],
privateKey: .privateKey(.forTesting())
)
)
var clientConfig = TLSConfiguration.makeClientConfiguration()
clientConfig.trustRoots = try .certificates([.forTesting()])
let clientContext = try NIOSSLContext(configuration: clientConfig)
let dummyAddress = try SocketAddress(ipAddress: "1.2.3.4", port: 5678)
let backToBack = BackToBackEmbeddedChannel()
let serverHandler = NIOSSLServerHandler(context: serverContext)
let clientHandler = try NIOSSLClientHandler(context: clientContext, serverHostname: "localhost")
try backToBack.client.pipeline.addHandler(clientHandler).wait()
try backToBack.server.pipeline.addHandler(serverHandler).wait()
// To trigger activation of both channels we use connect().
try backToBack.client.connect(to: dummyAddress).wait()
try backToBack.server.connect(to: dummyAddress).wait()
try backToBack.interactInMemory()
// Let's try 512 bytes.
var buffer = backToBack.client.allocator.buffer(capacity: 512)
buffer.writeBytes(repeatElement(0, count: 512))
for _ in 0..<writeCount {
// A vector of 100 writes.
for _ in 0..<100 {
backToBack.client.write(buffer, promise: nil)
}
backToBack.client.flush()
try backToBack.interactInMemory()
// Pull any data out of the server to avoid ballooning in memory.
while let _ = try backToBack.server.readInbound(as: ByteBuffer.self) {}
}
}
@@ -0,0 +1,52 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
import NIOCore
import NIOEmbedded
import NIOSSL
func runSimpleHandshake(handshakeCount: Int) throws {
let serverContext = try NIOSSLContext(
configuration: .makeServerConfiguration(
certificateChain: [.certificate(.forTesting())],
privateKey: .privateKey(.forTesting())
)
)
var clientConfig = TLSConfiguration.makeClientConfiguration()
clientConfig.trustRoots = try .certificates([.forTesting()])
let clientContext = try NIOSSLContext(configuration: clientConfig)
let dummyAddress = try SocketAddress(ipAddress: "1.2.3.4", port: 5678)
for _ in 0..<handshakeCount {
let backToBack = BackToBackEmbeddedChannel()
let serverHandler = NIOSSLServerHandler(context: serverContext)
let clientHandler = try NIOSSLClientHandler(context: clientContext, serverHostname: "localhost")
try backToBack.client.pipeline.addHandler(clientHandler).wait()
try backToBack.server.pipeline.addHandler(serverHandler).wait()
// To trigger activation of both channels we use connect().
try backToBack.client.connect(to: dummyAddress).wait()
try backToBack.server.connect(to: dummyAddress).wait()
try backToBack.interactInMemory()
// Ok, now do shutdown.
backToBack.client.close(promise: nil)
try backToBack.interactInMemory()
try backToBack.client.closeFuture.wait()
try backToBack.server.closeFuture.wait()
}
}
+30
View File
@@ -0,0 +1,30 @@
// swift-tools-version: 5.9
import PackageDescription
let package = Package(
name: "benchmarks",
platforms: [
.macOS("14")
],
dependencies: [
.package(path: "../"),
.package(url: "https://github.com/apple/swift-nio.git", from: "2.54.0"),
.package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.22.0"),
],
targets: [
.executableTarget(
name: "NIOSSHBenchmarks",
dependencies: [
.product(name: "Benchmark", package: "package-benchmark"),
.product(name: "NIOSSL", package: "swift-nio-ssl"),
.product(name: "NIOCore", package: "swift-nio"),
.product(name: "NIOEmbedded", package: "swift-nio"),
],
path: "Benchmarks/NIOSSLBenchmarks",
plugins: [
.plugin(name: "BenchmarkPlugin", package: "package-benchmark")
]
)
]
)
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 201948
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 644865
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 201948
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 644868
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 201947
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 643873
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 201947
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 643873
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 201947
}
@@ -0,0 +1,3 @@
{
"mallocCountTotal" : 643873
}
@@ -1,14 +0,0 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2019 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
@@ -1,54 +0,0 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2017-2019 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
# shellcheck source=IntegrationTests/tests_02_allocation_counters/defines.sh
source defines.sh
set -eu
here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
all_tests=()
for file in "$here/test_01_resources/"test_*.swift; do
test_name=$(basename "$file")
test_name=${test_name#test_*}
test_name=${test_name%*.swift}
all_tests+=( "$test_name" )
done
# shellcheck disable=SC2154 # Provided by framework
"$here/test_01_resources/run-nio-ssl-alloc-counter-tests.sh" -t "$tmp" > "$tmp/output"
for test in "${all_tests[@]}"; do
cat "$tmp/output" # helps debugging
total_allocations=$(grep "^test_$test.total_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
not_freed_allocations=$(grep "^test_$test.remaining_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
max_allowed_env_name="MAX_ALLOCS_ALLOWED_$test"
info "$test: allocations not freed: $not_freed_allocations"
info "$test: total number of mallocs: $total_allocations"
assert_less_than "$not_freed_allocations" 5 # allow some slack
assert_greater_than "$not_freed_allocations" -5 # allow some slack
assert_greater_than "$total_allocations" 1000
if [[ -z "${!max_allowed_env_name+x}" ]]; then
if [[ -z "${!max_allowed_env_name+x}" ]]; then
warn "no reference number of allocations set (set to \$$max_allowed_env_name)"
warn "to set current number:"
warn " export $max_allowed_env_name=$total_allocations"
fi
else
assert_less_than_or_equal "$total_allocations" "${!max_allowed_env_name}"
fi
done
@@ -1,55 +0,0 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
set -eu
here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
tmp_dir="/tmp"
while getopts "t:" opt; do
case "$opt" in
t)
tmp_dir="$OPTARG"
;;
*)
exit 1
;;
esac
done
nio_checkout=$(mktemp -d "$tmp_dir/.swift-nio_XXXXXX")
(
cd "$nio_checkout"
git clone --depth 1 https://github.com/apple/swift-nio
)
shift $((OPTIND-1))
tests_to_run=("$here"/test_*.swift)
if [[ $# -gt 0 ]]; then
tests_to_run=("$@")
fi
"$nio_checkout/swift-nio/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh" \
-p "$here/../../.." \
-m NIOCore \
-m NIOPosix \
-m NIOEmbedded \
-m NIOSSL \
-s "$here/shared.swift" \
-t "$tmp_dir" \
-d <( echo '.package(url: "https://github.com/apple/swift-nio.git", from: "2.32.0"),' ) \
"${tests_to_run[@]}"
@@ -1,64 +0,0 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
import NIOCore
import NIOEmbedded
import NIOSSL
func run(identifier: String) {
let serverContext = try! NIOSSLContext(
configuration: .makeServerConfiguration(
certificateChain: [.certificate(.forTesting())],
privateKey: .privateKey(.forTesting())
)
)
var clientConfig = TLSConfiguration.makeClientConfiguration()
clientConfig.trustRoots = try! .certificates([.forTesting()])
let clientContext = try! NIOSSLContext(configuration: clientConfig)
let dummyAddress = try! SocketAddress(ipAddress: "1.2.3.4", port: 5678)
let backToBack = BackToBackEmbeddedChannel()
let serverHandler = NIOSSLServerHandler(context: serverContext)
let clientHandler = try! NIOSSLClientHandler(context: clientContext, serverHostname: "localhost")
try! backToBack.client.pipeline.addHandler(clientHandler).wait()
try! backToBack.server.pipeline.addHandler(serverHandler).wait()
// To trigger activation of both channels we use connect().
try! backToBack.client.connect(to: dummyAddress).wait()
try! backToBack.server.connect(to: dummyAddress).wait()
try! backToBack.interactInMemory()
// Let's try 512 bytes.
var buffer = backToBack.client.allocator.buffer(capacity: 512)
buffer.writeBytes(repeatElement(0, count: 512))
measure(identifier: identifier) {
for _ in 0..<1000 {
// A vector of 100 writes.
for _ in 0..<100 {
backToBack.client.write(buffer, promise: nil)
}
backToBack.client.flush()
try! backToBack.interactInMemory()
// Pull any data out of the server to avoid ballooning in memory.
while let _ = try! backToBack.server.readInbound(as: ByteBuffer.self) {}
}
return 1000
}
}
@@ -1,56 +0,0 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
import NIOCore
import NIOEmbedded
import NIOSSL
func run(identifier: String) {
let serverContext = try! NIOSSLContext(
configuration: .makeServerConfiguration(
certificateChain: [.certificate(.forTesting())],
privateKey: .privateKey(.forTesting())
)
)
var clientConfig = TLSConfiguration.makeClientConfiguration()
clientConfig.trustRoots = try! .certificates([.forTesting()])
let clientContext = try! NIOSSLContext(configuration: clientConfig)
let dummyAddress = try! SocketAddress(ipAddress: "1.2.3.4", port: 5678)
measure(identifier: identifier) {
for _ in 0..<1000 {
let backToBack = BackToBackEmbeddedChannel()
let serverHandler = NIOSSLServerHandler(context: serverContext)
let clientHandler = try! NIOSSLClientHandler(context: clientContext, serverHostname: "localhost")
try! backToBack.client.pipeline.addHandler(clientHandler).wait()
try! backToBack.server.pipeline.addHandler(serverHandler).wait()
// To trigger activation of both channels we use connect().
try! backToBack.client.connect(to: dummyAddress).wait()
try! backToBack.server.connect(to: dummyAddress).wait()
try! backToBack.interactInMemory()
// Ok, now do shutdown.
backToBack.client.close(promise: nil)
try! backToBack.interactInMemory()
try! backToBack.client.closeFuture.wait()
try! backToBack.server.closeFuture.wait()
}
return 1000
}
}
-7
View File
@@ -9,13 +9,6 @@ services:
ubuntu_version: "jammy"
swift_version: "5.10"
test:
image: swift-nio-ssl:22.04-5.10
environment:
- MAX_ALLOCS_ALLOWED_simple_handshake=743000
- MAX_ALLOCS_ALLOWED_many_writes=201000
- IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error
performance-test:
image: swift-nio-ssl:22.04-5.10
-7
View File
@@ -9,13 +9,6 @@ services:
ubuntu_version: "jammy"
swift_version: "5.8"
test:
image: swift-nio-ssl:22.04-5.8
environment:
- MAX_ALLOCS_ALLOWED_simple_handshake=743000
- MAX_ALLOCS_ALLOWED_many_writes=201000
- IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error
performance-test:
image: swift-nio-ssl:22.04-5.8
-7
View File
@@ -9,13 +9,6 @@ services:
ubuntu_version: "jammy"
swift_version: "5.9"
test:
image: swift-nio-ssl:22.04-5.9
environment:
- MAX_ALLOCS_ALLOWED_simple_handshake=743000
- MAX_ALLOCS_ALLOWED_many_writes=201000
- IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error
performance-test:
image: swift-nio-ssl:22.04-5.9
-7
View File
@@ -8,13 +8,6 @@ services:
args:
base_image: "swiftlang/swift:nightly-main-jammy"
test:
image: swift-nio-ssl:22.04-main
environment:
- MAX_ALLOCS_ALLOWED_simple_handshake=743000
- MAX_ALLOCS_ALLOWED_many_writes=201000
- IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error
performance-test:
image: swift-nio-ssl:22.04-main