mirror of
https://github.com/apple/swift-nio
synced 2026-11-22 04:30:41 +00:00
### Motivation * External callers of `macos_benchmarks.yml` (e.g. swift-log) currently invoke the reusable workflow once per benchmark package, paying self-hosted macOS runner-recycling cost for each invocation. ### Modifications * Add a new `benchmark_package_paths` input to `.github/workflows/macos_benchmarks.yml` and mark the singular `benchmark_package_path` as deprecated. The run step forwards both inputs straight to the script via `BENCHMARK_PACKAGE_PATH` and `BENCHMARK_PACKAGE_PATHS` environment variables; the workflow itself does no path parsing. * Teach `scripts/check_benchmark_thresholds.sh` to accept `BENCHMARK_PACKAGE_PATHS` as either a JSON array of strings (auto-detected by a leading `[`, requires `jq`) or a newline-separated list, falling back to the legacy `BENCHMARK_PACKAGE_PATH` when unset or when the JSON form is an empty array. * The script now loops over the resolved package list, wraps each iteration in a GitHub Actions `::group::`, and emits an `::error::` summary listing failed packages at the end, exiting non-zero if any failed. ### Result * A single workflow invocation runs all listed packages serially on each Swift/Xcode runner, paying runner-recycling cost once per Swift version instead of once per (version x package). * Existing single-package callers keep working unchanged. * Adopters of the script in other contexts (e.g. the Linux `benchmarks.yml`) can pick up multi-package support by passing `BENCHMARK_PACKAGE_PATHS` without reimplementing parsing or failure-collection logic. An example of this working https://github.com/apple/swift-log/pull/466
107 lines
3.9 KiB
Bash
Executable File
107 lines
3.9 KiB
Bash
Executable File
#!/bin/bash
|
|
##===----------------------------------------------------------------------===##
|
|
##
|
|
## This source file is part of the SwiftNIO open source project
|
|
##
|
|
## Copyright (c) 2025 Apple Inc. and the SwiftNIO project authors
|
|
## Licensed under Apache License v2.0
|
|
##
|
|
## See LICENSE.txt for license information
|
|
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
|
|
##
|
|
## SPDX-License-Identifier: Apache-2.0
|
|
##
|
|
##===----------------------------------------------------------------------===##
|
|
|
|
set -uo pipefail
|
|
|
|
log() { printf -- "** %s\n" "$*" >&2; }
|
|
error() { printf -- "** ERROR: %s\n" "$*" >&2; }
|
|
fatal() { error "$@"; exit 1; }
|
|
|
|
# Parameter environment variables
|
|
if [ -z "$SWIFT_VERSION" ]; then
|
|
fatal "SWIFT_VERSION must be specified."
|
|
fi
|
|
|
|
# Accept either BENCHMARK_PACKAGE_PATHS (newline-separated list, or a JSON
|
|
# array of strings such as '["A", "B"]') or the legacy singular
|
|
# BENCHMARK_PACKAGE_PATH. The plural takes precedence; falling back to the
|
|
# singular preserves the original interface. The JSON form requires `jq`.
|
|
plural_paths="${BENCHMARK_PACKAGE_PATHS:-}"
|
|
singular_path="${BENCHMARK_PACKAGE_PATH:-.}"
|
|
|
|
trimmed="${plural_paths#"${plural_paths%%[![:space:]]*}"}"
|
|
if [[ "$trimmed" == \[* ]]; then
|
|
command -v jq >/dev/null 2>&1 || fatal "BENCHMARK_PACKAGE_PATHS looks like a JSON array but \`jq\` is not installed."
|
|
jq empty <<< "$plural_paths" 2>/dev/null || fatal "BENCHMARK_PACKAGE_PATHS is not valid JSON."
|
|
jq -e 'type == "array" and all(.[]; type == "string")' >/dev/null <<< "$plural_paths" \
|
|
|| fatal "BENCHMARK_PACKAGE_PATHS must be a JSON array of strings."
|
|
if [[ "$(jq 'length' <<< "$plural_paths")" == "0" ]]; then
|
|
benchmark_package_paths="$singular_path"
|
|
else
|
|
benchmark_package_paths=$(jq -r '.[]' <<< "$plural_paths")
|
|
fi
|
|
elif [[ -n "$plural_paths" ]]; then
|
|
benchmark_package_paths="$plural_paths"
|
|
else
|
|
benchmark_package_paths="$singular_path"
|
|
fi
|
|
|
|
swift_version="${SWIFT_VERSION:-""}"
|
|
|
|
# Any parameters to the script are passed along to SwiftPM
|
|
swift_package_arguments=("$@")
|
|
|
|
run_one() {
|
|
local benchmark_package_path="$1"
|
|
|
|
#"swift package --package-path ${{ inputs.benchmark_package_path }} ${{ inputs.swift_package_arguments }} benchmark baseline check --check-absolute-path ${{ inputs.benchmark_package_path }}/Thresholds/${SWIFT_VERSION}/"
|
|
swift package --package-path "$benchmark_package_path" "${swift_package_arguments[@]}" benchmark thresholds check --format metricP90AbsoluteThresholds --path "${benchmark_package_path}/Thresholds/${swift_version}/"
|
|
local rc="$?"
|
|
|
|
# Benchmarks are unchanged, nothing to recalculate
|
|
if [[ "$rc" == 0 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# Non-zero exit from 'thresholds check' means thresholds regressed or a build
|
|
# error occurred. Try 'thresholds update' to distinguish: if it also fails, it
|
|
# was a build error; if it succeeds, thresholds were updated.
|
|
log "Recalculating thresholds for ${benchmark_package_path}..."
|
|
|
|
swift package --package-path "$benchmark_package_path" "${swift_package_arguments[@]}" benchmark thresholds update --format metricP90AbsoluteThresholds --path "${benchmark_package_path}/Thresholds/${swift_version}/"
|
|
local update_rc="$?"
|
|
|
|
if [[ "$update_rc" != 0 ]]; then
|
|
error "Benchmark in ${benchmark_package_path} failed to run due to build error."
|
|
return "$update_rc"
|
|
fi
|
|
|
|
echo "=== BEGIN DIFF (${benchmark_package_path}) ===" # use echo, not log for clean output to be scraped
|
|
git add --intent-to-add "${benchmark_package_path}/Thresholds/"
|
|
git diff HEAD -- "${benchmark_package_path}/Thresholds/"
|
|
return 1
|
|
}
|
|
|
|
overall_rc=0
|
|
failed=()
|
|
while IFS= read -r path; do
|
|
[ -z "$path" ] && continue
|
|
echo "::group::Running benchmarks for $path"
|
|
run_one "$path"
|
|
rc=$?
|
|
echo "::endgroup::"
|
|
if [[ "$rc" -ne 0 ]]; then
|
|
overall_rc=$rc
|
|
failed+=("$path")
|
|
fi
|
|
done <<< "$benchmark_package_paths"
|
|
|
|
if [[ "$overall_rc" -ne 0 ]]; then
|
|
echo "::error::Benchmark failures in: ${failed[*]}"
|
|
fi
|
|
exit "$overall_rc"
|
|
|
|
|