From 75e5a978e83696067fb0114c563fafbe8d2ae330 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Fri, 25 Oct 2024 16:17:41 +0100 Subject: [PATCH] Switch allocation benchmarks to GHA (#488) ### Motivation: This will allow us to decommission the old CI hooks. ### Modifications: * Migrate the existing tests to use the package-benchmark framework. * Delete the obsolete integration tests * Remove allocations test references from the docker files. ### Result: Allocation counts are checked using GitHub Actions. --- .github/workflows/pull_request.yml | 8 ++- .github/workflows/scheduled.yml | 6 ++ .../NIOSSLBenchmarks/Benchmarks.swift | 54 ++++++++++++++++ .../NIOSSLBenchmarks/ManyWrites.swift | 60 +++++++++++++++++ .../Benchmarks/NIOSSLBenchmarks/Shared.swift | 0 .../NIOSSLBenchmarks/SimpleHandshake.swift | 52 +++++++++++++++ Benchmarks/Package.swift | 30 +++++++++ .../5.10/NIOSSHBenchmarks.ManyWrites.p90.json | 3 + .../NIOSSHBenchmarks.SimpleHandshake.p90.json | 3 + .../5.9/NIOSSHBenchmarks.ManyWrites.p90.json | 3 + .../NIOSSHBenchmarks.SimpleHandshake.p90.json | 3 + .../6.0/NIOSSHBenchmarks.ManyWrites.p90.json | 3 + .../NIOSSHBenchmarks.SimpleHandshake.p90.json | 3 + .../NIOSSHBenchmarks.ManyWrites.p90.json | 3 + .../NIOSSHBenchmarks.SimpleHandshake.p90.json | 3 + .../NIOSSHBenchmarks.ManyWrites.p90.json | 3 + .../NIOSSHBenchmarks.SimpleHandshake.p90.json | 3 + .../tests_02_allocation_counters/defines.sh | 14 ---- .../test_01_allocation_counts.sh | 54 ---------------- .../run-nio-ssl-alloc-counter-tests.sh | 55 ---------------- .../test_01_resources/test_many_writes.swift | 64 ------------------- .../test_simple_handshake.swift | 56 ---------------- docker/docker-compose.2204.510.yaml | 7 -- docker/docker-compose.2204.58.yaml | 7 -- docker/docker-compose.2204.59.yaml | 7 -- docker/docker-compose.2204.main.yaml | 7 -- 26 files changed, 239 insertions(+), 272 deletions(-) create mode 100644 Benchmarks/Benchmarks/NIOSSLBenchmarks/Benchmarks.swift create mode 100644 Benchmarks/Benchmarks/NIOSSLBenchmarks/ManyWrites.swift rename IntegrationTests/tests_02_allocation_counters/test_01_resources/shared.swift => Benchmarks/Benchmarks/NIOSSLBenchmarks/Shared.swift (100%) create mode 100644 Benchmarks/Benchmarks/NIOSSLBenchmarks/SimpleHandshake.swift create mode 100644 Benchmarks/Package.swift create mode 100644 Benchmarks/Thresholds/5.10/NIOSSHBenchmarks.ManyWrites.p90.json create mode 100644 Benchmarks/Thresholds/5.10/NIOSSHBenchmarks.SimpleHandshake.p90.json create mode 100644 Benchmarks/Thresholds/5.9/NIOSSHBenchmarks.ManyWrites.p90.json create mode 100644 Benchmarks/Thresholds/5.9/NIOSSHBenchmarks.SimpleHandshake.p90.json create mode 100644 Benchmarks/Thresholds/6.0/NIOSSHBenchmarks.ManyWrites.p90.json create mode 100644 Benchmarks/Thresholds/6.0/NIOSSHBenchmarks.SimpleHandshake.p90.json create mode 100644 Benchmarks/Thresholds/nightly-6.0/NIOSSHBenchmarks.ManyWrites.p90.json create mode 100644 Benchmarks/Thresholds/nightly-6.0/NIOSSHBenchmarks.SimpleHandshake.p90.json create mode 100644 Benchmarks/Thresholds/nightly-main/NIOSSHBenchmarks.ManyWrites.p90.json create mode 100644 Benchmarks/Thresholds/nightly-main/NIOSSHBenchmarks.SimpleHandshake.p90.json delete mode 100644 IntegrationTests/tests_02_allocation_counters/defines.sh delete mode 100644 IntegrationTests/tests_02_allocation_counters/test_01_allocation_counts.sh delete mode 100755 IntegrationTests/tests_02_allocation_counters/test_01_resources/run-nio-ssl-alloc-counter-tests.sh delete mode 100644 IntegrationTests/tests_02_allocation_counters/test_01_resources/test_many_writes.swift delete mode 100644 IntegrationTests/tests_02_allocation_counters/test_01_resources/test_simple_handshake.swift diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 284633f3..c2bd0484 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -25,7 +25,13 @@ jobs: uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main with: name: "Integration test" - matrix_linux_command: "apt-get update -yq && apt-get install -yq execstack lsof dnsutils netcat-openbsd net-tools expect curl jq && ./scripts/integration_tests.sh -f test_01_renegotiation" + matrix_linux_command: "apt-get update -yq && apt-get install -yq execstack lsof dnsutils netcat-openbsd net-tools expect curl jq && ./scripts/integration_tests.sh" + + benchmarks: + name: Benchmarks + uses: apple/swift-nio/.github/workflows/benchmarks.yml@main + with: + benchmark_package_path: "Benchmarks" cxx-interop: name: Cxx interop diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 12e8e536..6c41ef22 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -22,3 +22,9 @@ jobs: with: name: "Integration test" matrix_linux_command: "apt-get update -yq && apt-get install -yq execstack lsof dnsutils netcat-openbsd net-tools expect curl jq && ./scripts/integration_tests.sh -f test_01_renegotiation" + + benchmarks: + name: Benchmarks + uses: apple/swift-nio/.github/workflows/benchmarks.yml@main + with: + benchmark_package_path: "Benchmarks" diff --git a/Benchmarks/Benchmarks/NIOSSLBenchmarks/Benchmarks.swift b/Benchmarks/Benchmarks/NIOSSLBenchmarks/Benchmarks.swift new file mode 100644 index 00000000..61ffed9a --- /dev/null +++ b/Benchmarks/Benchmarks/NIOSSLBenchmarks/Benchmarks.swift @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import NIOCore +import NIOEmbedded +import NIOSSL + +let benchmarks = { + let defaultMetrics: [BenchmarkMetric] = [ + .mallocCountTotal + ] + + Benchmark( + "SimpleHandshake", + configuration: .init( + metrics: defaultMetrics, + scalingFactor: .kilo, + maxDuration: .seconds(10_000_000), + maxIterations: 10, + thresholds: [.mallocCountTotal: .init(absolute: [.p90: 2000])] + ) + ) { benchmark in + try runSimpleHandshake( + handshakeCount: benchmark.scaledIterations.upperBound + ) + } + + Benchmark( + "ManyWrites", + configuration: .init( + metrics: defaultMetrics, + scalingFactor: .kilo, + maxDuration: .seconds(10_000_000), + maxIterations: 10, + thresholds: [.mallocCountTotal: .init(absolute: [.p90: 2000])] + ) + ) { benchmark in + try runManyWrites( + writeCount: benchmark.scaledIterations.upperBound + ) + } +} diff --git a/Benchmarks/Benchmarks/NIOSSLBenchmarks/ManyWrites.swift b/Benchmarks/Benchmarks/NIOSSLBenchmarks/ManyWrites.swift new file mode 100644 index 00000000..594e6ad1 --- /dev/null +++ b/Benchmarks/Benchmarks/NIOSSLBenchmarks/ManyWrites.swift @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore +import NIOEmbedded +import NIOSSL + +func runManyWrites(writeCount: Int) throws { + let serverContext = try NIOSSLContext( + configuration: .makeServerConfiguration( + certificateChain: [.certificate(.forTesting())], + privateKey: .privateKey(.forTesting()) + ) + ) + + var clientConfig = TLSConfiguration.makeClientConfiguration() + clientConfig.trustRoots = try .certificates([.forTesting()]) + let clientContext = try NIOSSLContext(configuration: clientConfig) + + let dummyAddress = try SocketAddress(ipAddress: "1.2.3.4", port: 5678) + let backToBack = BackToBackEmbeddedChannel() + let serverHandler = NIOSSLServerHandler(context: serverContext) + let clientHandler = try NIOSSLClientHandler(context: clientContext, serverHostname: "localhost") + try backToBack.client.pipeline.addHandler(clientHandler).wait() + try backToBack.server.pipeline.addHandler(serverHandler).wait() + + // To trigger activation of both channels we use connect(). + try backToBack.client.connect(to: dummyAddress).wait() + try backToBack.server.connect(to: dummyAddress).wait() + + try backToBack.interactInMemory() + + // Let's try 512 bytes. + var buffer = backToBack.client.allocator.buffer(capacity: 512) + buffer.writeBytes(repeatElement(0, count: 512)) + + for _ in 0.. "$tmp/output" - -for test in "${all_tests[@]}"; do - cat "$tmp/output" # helps debugging - total_allocations=$(grep "^test_$test.total_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g') - not_freed_allocations=$(grep "^test_$test.remaining_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g') - max_allowed_env_name="MAX_ALLOCS_ALLOWED_$test" - - info "$test: allocations not freed: $not_freed_allocations" - info "$test: total number of mallocs: $total_allocations" - - assert_less_than "$not_freed_allocations" 5 # allow some slack - assert_greater_than "$not_freed_allocations" -5 # allow some slack - assert_greater_than "$total_allocations" 1000 - if [[ -z "${!max_allowed_env_name+x}" ]]; then - if [[ -z "${!max_allowed_env_name+x}" ]]; then - warn "no reference number of allocations set (set to \$$max_allowed_env_name)" - warn "to set current number:" - warn " export $max_allowed_env_name=$total_allocations" - fi - else - assert_less_than_or_equal "$total_allocations" "${!max_allowed_env_name}" - fi -done diff --git a/IntegrationTests/tests_02_allocation_counters/test_01_resources/run-nio-ssl-alloc-counter-tests.sh b/IntegrationTests/tests_02_allocation_counters/test_01_resources/run-nio-ssl-alloc-counter-tests.sh deleted file mode 100755 index 5b748d54..00000000 --- a/IntegrationTests/tests_02_allocation_counters/test_01_resources/run-nio-ssl-alloc-counter-tests.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftNIO open source project -## -## Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftNIO project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -set -eu -here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -tmp_dir="/tmp" - -while getopts "t:" opt; do - case "$opt" in - t) - tmp_dir="$OPTARG" - ;; - *) - exit 1 - ;; - esac -done - -nio_checkout=$(mktemp -d "$tmp_dir/.swift-nio_XXXXXX") -( -cd "$nio_checkout" -git clone --depth 1 https://github.com/apple/swift-nio -) - -shift $((OPTIND-1)) - -tests_to_run=("$here"/test_*.swift) - -if [[ $# -gt 0 ]]; then - tests_to_run=("$@") -fi - -"$nio_checkout/swift-nio/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh" \ - -p "$here/../../.." \ - -m NIOCore \ - -m NIOPosix \ - -m NIOEmbedded \ - -m NIOSSL \ - -s "$here/shared.swift" \ - -t "$tmp_dir" \ - -d <( echo '.package(url: "https://github.com/apple/swift-nio.git", from: "2.32.0"),' ) \ - "${tests_to_run[@]}" diff --git a/IntegrationTests/tests_02_allocation_counters/test_01_resources/test_many_writes.swift b/IntegrationTests/tests_02_allocation_counters/test_01_resources/test_many_writes.swift deleted file mode 100644 index f984ac73..00000000 --- a/IntegrationTests/tests_02_allocation_counters/test_01_resources/test_many_writes.swift +++ /dev/null @@ -1,64 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the SwiftNIO open source project -// -// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of SwiftNIO project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import NIOCore -import NIOEmbedded -import NIOSSL - -func run(identifier: String) { - let serverContext = try! NIOSSLContext( - configuration: .makeServerConfiguration( - certificateChain: [.certificate(.forTesting())], - privateKey: .privateKey(.forTesting()) - ) - ) - - var clientConfig = TLSConfiguration.makeClientConfiguration() - clientConfig.trustRoots = try! .certificates([.forTesting()]) - let clientContext = try! NIOSSLContext(configuration: clientConfig) - - let dummyAddress = try! SocketAddress(ipAddress: "1.2.3.4", port: 5678) - let backToBack = BackToBackEmbeddedChannel() - let serverHandler = NIOSSLServerHandler(context: serverContext) - let clientHandler = try! NIOSSLClientHandler(context: clientContext, serverHostname: "localhost") - try! backToBack.client.pipeline.addHandler(clientHandler).wait() - try! backToBack.server.pipeline.addHandler(serverHandler).wait() - - // To trigger activation of both channels we use connect(). - try! backToBack.client.connect(to: dummyAddress).wait() - try! backToBack.server.connect(to: dummyAddress).wait() - - try! backToBack.interactInMemory() - - // Let's try 512 bytes. - var buffer = backToBack.client.allocator.buffer(capacity: 512) - buffer.writeBytes(repeatElement(0, count: 512)) - - measure(identifier: identifier) { - for _ in 0..<1000 { - // A vector of 100 writes. - for _ in 0..<100 { - backToBack.client.write(buffer, promise: nil) - } - backToBack.client.flush() - - try! backToBack.interactInMemory() - - // Pull any data out of the server to avoid ballooning in memory. - while let _ = try! backToBack.server.readInbound(as: ByteBuffer.self) {} - } - - return 1000 - } -} diff --git a/IntegrationTests/tests_02_allocation_counters/test_01_resources/test_simple_handshake.swift b/IntegrationTests/tests_02_allocation_counters/test_01_resources/test_simple_handshake.swift deleted file mode 100644 index 1b11300d..00000000 --- a/IntegrationTests/tests_02_allocation_counters/test_01_resources/test_simple_handshake.swift +++ /dev/null @@ -1,56 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the SwiftNIO open source project -// -// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of SwiftNIO project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import NIOCore -import NIOEmbedded -import NIOSSL - -func run(identifier: String) { - let serverContext = try! NIOSSLContext( - configuration: .makeServerConfiguration( - certificateChain: [.certificate(.forTesting())], - privateKey: .privateKey(.forTesting()) - ) - ) - - var clientConfig = TLSConfiguration.makeClientConfiguration() - clientConfig.trustRoots = try! .certificates([.forTesting()]) - let clientContext = try! NIOSSLContext(configuration: clientConfig) - - let dummyAddress = try! SocketAddress(ipAddress: "1.2.3.4", port: 5678) - - measure(identifier: identifier) { - for _ in 0..<1000 { - let backToBack = BackToBackEmbeddedChannel() - let serverHandler = NIOSSLServerHandler(context: serverContext) - let clientHandler = try! NIOSSLClientHandler(context: clientContext, serverHostname: "localhost") - try! backToBack.client.pipeline.addHandler(clientHandler).wait() - try! backToBack.server.pipeline.addHandler(serverHandler).wait() - - // To trigger activation of both channels we use connect(). - try! backToBack.client.connect(to: dummyAddress).wait() - try! backToBack.server.connect(to: dummyAddress).wait() - - try! backToBack.interactInMemory() - - // Ok, now do shutdown. - backToBack.client.close(promise: nil) - try! backToBack.interactInMemory() - try! backToBack.client.closeFuture.wait() - try! backToBack.server.closeFuture.wait() - } - - return 1000 - } -} diff --git a/docker/docker-compose.2204.510.yaml b/docker/docker-compose.2204.510.yaml index 36a21f98..9dbf6d2a 100644 --- a/docker/docker-compose.2204.510.yaml +++ b/docker/docker-compose.2204.510.yaml @@ -9,13 +9,6 @@ services: ubuntu_version: "jammy" swift_version: "5.10" - test: - image: swift-nio-ssl:22.04-5.10 - environment: - - MAX_ALLOCS_ALLOWED_simple_handshake=743000 - - MAX_ALLOCS_ALLOWED_many_writes=201000 - - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error - performance-test: image: swift-nio-ssl:22.04-5.10 diff --git a/docker/docker-compose.2204.58.yaml b/docker/docker-compose.2204.58.yaml index 35eb494d..04e0b1c6 100644 --- a/docker/docker-compose.2204.58.yaml +++ b/docker/docker-compose.2204.58.yaml @@ -9,13 +9,6 @@ services: ubuntu_version: "jammy" swift_version: "5.8" - test: - image: swift-nio-ssl:22.04-5.8 - environment: - - MAX_ALLOCS_ALLOWED_simple_handshake=743000 - - MAX_ALLOCS_ALLOWED_many_writes=201000 - - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error - performance-test: image: swift-nio-ssl:22.04-5.8 diff --git a/docker/docker-compose.2204.59.yaml b/docker/docker-compose.2204.59.yaml index a9b9472b..d698ef4b 100644 --- a/docker/docker-compose.2204.59.yaml +++ b/docker/docker-compose.2204.59.yaml @@ -9,13 +9,6 @@ services: ubuntu_version: "jammy" swift_version: "5.9" - test: - image: swift-nio-ssl:22.04-5.9 - environment: - - MAX_ALLOCS_ALLOWED_simple_handshake=743000 - - MAX_ALLOCS_ALLOWED_many_writes=201000 - - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error - performance-test: image: swift-nio-ssl:22.04-5.9 diff --git a/docker/docker-compose.2204.main.yaml b/docker/docker-compose.2204.main.yaml index 9330020b..b82db6d9 100644 --- a/docker/docker-compose.2204.main.yaml +++ b/docker/docker-compose.2204.main.yaml @@ -8,13 +8,6 @@ services: args: base_image: "swiftlang/swift:nightly-main-jammy" - test: - image: swift-nio-ssl:22.04-main - environment: - - MAX_ALLOCS_ALLOWED_simple_handshake=743000 - - MAX_ALLOCS_ALLOWED_many_writes=201000 - - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error - performance-test: image: swift-nio-ssl:22.04-main