add the first allocation tests for nio-http2 (#122)

Motivation:

Allocation tests are good :).

Modifications:

Add two allocations tests and NIO's integration test framework.

Result:

Hopefully fewer allocations in the future. And better ways to check what
allocations are happening.
This commit is contained in:
Johannes Weiss authored and Cory Benfield committed 2019-05-17 14:36:46 +01:00
1 parent 6ab190327d
commit 16fae8a3d1
12 files changed
+682

No files matched your search

+58
View File
@@ -0,0 +1,58 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
function plugin_echo_test_suite_begin() {
echo "Running test suite '$1'"
}
function plugin_echo_test_suite_end() {
true
}
# test_name
function plugin_echo_test_begin() {
echo -n "Running test '$1'... "
}
function plugin_echo_test_skip() {
echo "Skipping test '$1'"
}
function plugin_echo_test_ok() {
echo "OK (${1}s)"
}
function plugin_echo_test_fail() {
echo "FAILURE ($1)"
echo "--- OUTPUT BEGIN ---"
cat "$2"
echo "--- OUTPUT END ---"
}
function plugin_echo_test_end() {
true
}
function plugin_echo_summary_ok() {
echo "OK (ran $1 tests successfully)"
}
function plugin_echo_summary_fail() {
echo "FAILURE (oks: $1, failures: $2)"
}
function plugin_echo_init() {
true
}
+119
View File
@@ -0,0 +1,119 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
junit_testsuite_time=0
function junit_output_write() {
extra_flags=""
if [[ "$1" == "-n" ]]; then
extra_flags="-n"
shift
fi
test -n "$junit_xml_output"
echo $extra_flags "$*" >> "$junit_xml_output"
}
function junit_output_cat() {
cat "$@" >> "$junit_xml_output"
}
# search, replace
function junit_output_replace() {
test -n "$junit_xml_output"
case "$(uname -s)" in
Linux)
sed -i "s/$1/$2/g" "$junit_xml_output"
;;
*)
sed -i "" "s/$1/$2/g" "$junit_xml_output"
;;
esac
}
function plugin_junit_xml_test_suite_begin() {
junit_testsuite_time=0
junit_output_write "<testsuite name='$1' hostname='$(hostname)' "\
"timestamp='$(date -u +"%Y-%m-%dT%H:%M:%S")' tests='XXX-TESTS-XXX' "\
"failures='XXX-FAILURES-XXX' time='XXX-TIME-XXX' errors='0' id='$(date +%s)'"\
" package='NIOIntegrationTests.$1'>"
}
function plugin_junit_xml_test_suite_end() {
junit_repl_success_and_fail "$1" "$2"
junit_output_write "</testsuite>"
}
# test_name
function plugin_junit_xml_test_begin() {
junit_output_write -n " <testcase classname='NIOIntegrationTests.$2' name='$1'"
}
function plugin_junit_xml_test_skip() {
true
}
function plugin_junit_xml_test_ok() {
time_ms=$1
junit_output_write " time='$time_ms'>"
junit_testsuite_time=$((junit_testsuite_time + time_ms))
}
function plugin_junit_xml_test_fail() {
time_ms=$1
junit_output_write " time='$time_ms'>"
junit_output_write " <failure type='test_fail'>"
junit_output_write " <system-out>"
junit_output_write ' <![CDATA['
junit_output_cat "$2"
junit_output_write ' ]]>'
junit_output_write " </system-out>"
junit_output_write " </failure>"
}
function plugin_junit_xml_test_end() {
junit_output_write " </testcase>"
}
function junit_repl_success_and_fail() {
junit_output_replace XXX-TESTS-XXX "$(($1 + $2))"
junit_output_replace XXX-FAILURES-XXX "$2"
junit_output_replace XXX-TIME-XXX "$junit_testsuite_time"
}
function plugin_junit_xml_summary_ok() {
junit_output_write "</testsuites>"
}
function plugin_junit_xml_summary_fail() {
junit_output_write "</testsuites>"
}
function plugin_junit_xml_init() {
junit_xml_output=""
for f in "$@"; do
if [[ "$junit_xml_output" = "PLACEHOLDER" ]]; then
junit_xml_output="$f"
fi
if [[ "$f" == "--junit-xml" && -z "$junit_xml_output" ]]; then
junit_xml_output="PLACEHOLDER"
fi
done
if [[ -z "$junit_xml_output" || "$junit_xml_output" = "PLACEHOLDER" ]]; then
echo >&2 "ERROR: you need to specify the output after the --junit-xml argument"
false
fi
echo "<testsuites>" > "$junit_xml_output"
}
+33
View File
@@ -0,0 +1,33 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
(
# this sub-shell is where the actual test is run
set -eu
set -x
set -o pipefail
test="$1"
tmp="$2"
root="$3"
g_show_info="$4"
here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "$here/test_functions.sh"
source "$test"
wait
)
exit_code=$?
exit $exit_code
+159
View File
@@ -0,0 +1,159 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
set -eu
shopt -s nullglob
here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
tmp=$(mktemp -d /tmp/.swift-nio-http1-server-sh-tests_XXXXXX)
# start_time
function time_diff_to_now() {
echo "$(( $(date +%s) - $1 ))"
}
function plugins_do() {
local method
method="$1"
shift
for plugin in $plugins; do
cd "$orig_cwd"
"plugin_${plugin}_${method}" "$@"
cd - > /dev/null
done
}
source "$here/plugin_echo.sh"
source "$here/plugin_junit_xml.sh"
plugins="echo"
plugin_opts_ind=0
if [[ "${1-default}" == "--junit-xml" ]]; then
plugins="echo junit_xml"
plugin_opts_ind=2
fi
function usage() {
echo >&2 "Usage: $0 [OPTIONS]"
echo >&2
echo >&2 "OPTIONS:"
echo >&2 " -f FILTER: Only run tests matching FILTER (regex)"
}
orig_cwd=$(pwd)
cd "$here"
plugins_do init "$@"
shift $plugin_opts_ind
filter="."
verbose=false
show_info=false
debug=false
while getopts "f:vid" opt; do
case $opt in
f)
filter="$OPTARG"
;;
v)
verbose=true
;;
i)
show_info=true
;;
d)
debug=true
;;
\?)
usage
exit 1
;;
esac
done
function run_test() {
if $verbose; then
"$@" 2>&1 | tee -a "$out"
# we need to return the return value of the first command
return ${PIPESTATUS[0]}
else
"$@" >> "$out" 2>&1
fi
}
exec 3>&1 4>&2 # copy stdout/err to fd 3/4 to we can output control messages
cnt_ok=0
cnt_fail=0
for f in tests_*; do
suite_ok=0
suite_fail=0
plugins_do test_suite_begin "$f"
start_suite=$(date +%s)
cd "$f"
for t in test_*.sh; do
if [[ ! "$f/$t" =~ $filter ]]; then
plugins_do test_skip "$t"
continue
fi
out=$(mktemp "$tmp/test.out_XXXXXX")
test_tmp=$(mktemp -d "$tmp/test.tmp_XXXXXX")
plugins_do test_begin "$t" "$f"
start=$(date +%s)
if run_test "$here/run-single-test.sh" "$here/$f/$t" "$test_tmp" "$here/.." "$show_info"; then
plugins_do test_ok "$(time_diff_to_now $start)"
suite_ok=$((suite_ok+1))
if $verbose; then
cat "$out"
fi
else
plugins_do test_fail "$(time_diff_to_now $start)" "$out"
suite_fail=$((suite_fail+1))
fi
if ! $debug; then
rm "$out"
rm -rf "$test_tmp"
fi
plugins_do test_end
done
cnt_ok=$((cnt_ok + suite_ok))
cnt_fail=$((cnt_fail + suite_fail))
cd ..
plugins_do test_suite_end "$(time_diff_to_now $start_suite)" "$suite_ok" "$suite_fail"
done
if ! $debug; then
rm -rf "$tmp"
else
echo >&2 "debug mode, not deleting '$tmp'"
fi
# report
if [[ $cnt_fail > 0 ]]; then
# kill leftovers (the whole process group)
trap '' TERM
kill 0
plugins_do summary_fail "$cnt_ok" "$cnt_fail"
else
plugins_do summary_ok "$cnt_ok" "$cnt_fail"
fi
if [[ $cnt_fail > 0 ]]; then
exit 1
else
exit 0
fi
+78
View File
@@ -0,0 +1,78 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
function fail() {
echo >&2 "FAILURE: $*"
false
}
function assert_equal() {
if [[ "$1" != "$2" ]]; then
fail "expected '$1', got '$2' ${3-}"
fi
}
function assert_equal_files() {
if ! cmp -s "$1" "$2"; then
diff -u "$1" "$2" || true
echo
echo "--- SNIP ($1, size=$(wc "$1"), SHA=$(shasum "$1")) ---"
cat "$1"
echo "--- SNAP ($1)---"
echo "--- SNIP ($2, size=$(wc "$2"), SHA=$(shasum "$2")) ---"
cat "$2"
echo "--- SNAP ($2) ---"
fail "file '$1' not equal to '$2'"
fi
}
function assert_less_than() {
if [[ ! "$1" -lt "$2" ]]; then
fail "assertion '$1' < '$2' failed"
fi
}
function assert_less_than_or_equal() {
if [[ ! "$1" -le "$2" ]]; then
fail "assertion '$1' <= '$2' failed"
fi
}
function assert_greater_than() {
if [[ ! "$1" -gt "$2" ]]; then
fail "assertion '$1' > '$2' failed"
fi
}
function assert_greater_than_or_equal() {
if [[ ! "$1" -ge "$2" ]]; then
fail "assertion '$1' >= '$2' failed"
fi
}
g_has_previously_infoed=false
function info() {
if $g_show_info; then
if ! $g_has_previously_infoed; then
echo >&3 || true # echo an extra newline so it looks better
g_has_previously_infoed=true
fi
echo >&3 "info: $*" || true
fi
}
function warn() {
echo >&4 "warning: $*"
}
@@ -0,0 +1,14 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
@@ -0,0 +1,52 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
source defines.sh
set -eu
here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
all_tests=()
for file in "$here/test_01_resources/"test_*.swift; do
test_name=$(basename "$file")
test_name=${test_name#test_*}
test_name=${test_name%*.swift}
all_tests+=( "$test_name" )
done
"$here/test_01_resources/run-nio-http2-alloc-counter-tests.sh" -t "$tmp" > "$tmp/output"
for test in "${all_tests[@]}"; do
cat "$tmp/output" # helps debugging
total_allocations=$(grep "^test_$test.total_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
not_freed_allocations=$(grep "^test_$test.remaining_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
max_allowed_env_name="MAX_ALLOCS_ALLOWED_$test"
info "$test: allocations not freed: $not_freed_allocations"
info "$test: total number of mallocs: $total_allocations"
assert_less_than "$not_freed_allocations" 5 # allow some slack
assert_greater_than "$not_freed_allocations" -5 # allow some slack
assert_greater_than "$total_allocations" 1000
if [[ -z "${!max_allowed_env_name+x}" ]]; then
if [[ -z "${!max_allowed_env_name+x}" ]]; then
warn "no reference number of allocations set (set to \$$max_allowed_env_name)"
warn "to set current number:"
warn " export $max_allowed_env_name=$total_allocations"
fi
else
assert_less_than_or_equal "$total_allocations" "${!max_allowed_env_name}"
fi
done
@@ -0,0 +1,50 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2019 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
set -eu
here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
tmp_dir="/tmp"
function die() {
echo >&2 "ERROR: $*"
exit 1
}
while getopts "t:" opt; do
case "$opt" in
t)
tmp_dir="$OPTARG"
;;
\?)
die "unknown option $opt"
;;
esac
done
nio_checkout=$(mktemp -d "$tmp_dir/.swift-nio_XXXXXX")
(
cd "$nio_checkout"
git clone --depth 1 https://github.com/apple/swift-nio
)
"$nio_checkout/swift-nio/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh" \
-p "$here/../../.." \
-m NIO \
-m NIOHTTP1 \
-m NIOHTTP2 \
-t "$tmp_dir" \
-d <( echo '.package(url: "https://github.com/apple/swift-nio.git", from: "2.0.0"),' ) \
"$here"/test_*.swift
@@ -0,0 +1,46 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2019 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
import NIO
import NIOHPACK
import NIOHTTP1
import NIOHTTP2
func run(identifier: String) {
let channel = EmbeddedChannel(handler: NIOHTTP2Handler(mode: .client))
let multiplexer = HTTP2StreamMultiplexer(mode: .client, channel: channel)
try! channel.pipeline.addHandler(multiplexer).wait()
try! channel.connect(to: SocketAddress(ipAddress: "1.2.3.4", port: 5678)).wait()
measure(identifier: identifier) {
var sumOfStreamIDs = 0
for _ in 0..<1000 {
let promise = channel.eventLoop.makePromise(of: Channel.self)
multiplexer.createStreamChannel(promise: promise) { (channel, streamID) in
return channel.eventLoop.makeSucceededFuture(())
}
channel.embeddedEventLoop.run()
let child = try! promise.futureResult.wait()
let streamID = try! Int(child.getOption(HTTP2StreamChannelOptions.streamID).wait())
sumOfStreamIDs += streamID
let closeFuture = child.close()
channel.embeddedEventLoop.run()
try! closeFuture.wait()
}
return sumOfStreamIDs
}
}
@@ -0,0 +1,46 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2019 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
import NIO
import NIOHTTP2
import NIOHPACK
func run(identifier: String) {
var buffer = ByteBufferAllocator().buffer(capacity: 128)
buffer.writeBytes([0x82, 0x86, 0x84, 0x41, 0x0f, 0x77, 0x77, 0x77, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d])
let request1 = buffer
buffer.clear()
buffer.writeBytes([0x82, 0x86, 0x84, 0xbe, 0x58, 0x08, 0x6e, 0x6f, 0x2d, 0x63, 0x61, 0x63, 0x68, 0x65])
let request2 = buffer
buffer.clear()
buffer.writeBytes([0x82, 0x87, 0x85, 0xbf, 0x40, 0x0a, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x2d, 0x6b, 0x65, 0x79, 0x0c, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x2d, 0x76, 0x61, 0x6c, 0x75, 0x65])
let request3 = buffer
measure(identifier: identifier) {
var sum = 0
for _ in 0..<1000 {
var request1Buffer = request1
var request2Buffer = request2
var request3Buffer = request3
var decoder = HPACKDecoder(allocator: ByteBufferAllocator())
let decoded1 = try! decoder.decodeHeaders(from: &request1Buffer)
sum += decoded1.count
let decoded2 = try! decoder.decodeHeaders(from: &request2Buffer)
sum += decoded2.count
let decoded3 = try! decoder.decodeHeaders(from: &request3Buffer)
sum += decoded3.count
}
return sum
}
}
+10
View File
@@ -10,8 +10,18 @@ services:
swift_version: "5.0"
h2spec_version: "2.2.1"
integration-tests:
image: swift-nio-http2:18.04-5.0
environment:
- MAX_ALLOCS_ALLOWED_create_client_stream_channel=70010
- MAX_ALLOCS_ALLOWED_hpack_decoding=5050
test:
image: swift-nio-http2:18.04-5.0
command: /bin/bash -cl "swift test && ./scripts/integration_tests.sh"
environment:
- MAX_ALLOCS_ALLOWED_create_client_stream_channel=70010
- MAX_ALLOCS_ALLOWED_hpack_decoding=5050
h2spec:
image: swift-nio-http2:18.04-5.0
+17
View File
@@ -0,0 +1,17 @@
#!/bin/bash
##===----------------------------------------------------------------------===##
##
## This source file is part of the SwiftNIO open source project
##
## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
##
## SPDX-License-Identifier: Apache-2.0
##
##===----------------------------------------------------------------------===##
mkdir -p .build # for the junit.xml file
./IntegrationTests/run-tests.sh --junit-xml .build/junit-sh-tests.xml -i