107 lines
3.5 KiB
YAML
107 lines
3.5 KiB
YAML
name: benchmark-run
|
|
|
|
# Runs the committed benchmark (`just bench`) and records the numbers.
|
|
#
|
|
# This is a measurement, not a test: nothing here fails the build. Unlike
|
|
# benchmark-ab, which compares two refs inside one job, this just reports the
|
|
# absolute throughput of the current ref alongside Crow for reference.
|
|
#
|
|
# Absolute req/s is only meaningful against other runs on the same runner type,
|
|
# so compare like with like when reading the history.
|
|
#
|
|
# Non-SSL only. Windows is excluded: benchmark/Makefile depends on `nc`, `&`
|
|
# and `kill`, so it would need a PowerShell rewrite first.
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
duration:
|
|
description: "Load duration per server"
|
|
required: false
|
|
default: "5s"
|
|
connections:
|
|
description: "Concurrent connections"
|
|
required: false
|
|
default: "10"
|
|
crow:
|
|
description: "Also benchmark Crow v1.3.1 for reference"
|
|
type: boolean
|
|
required: false
|
|
default: true
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
bench:
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
os: [ubuntu-latest, macos-latest]
|
|
runs-on: ${{ matrix.os }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@v4
|
|
|
|
# macos runners ship without Go.
|
|
- name: setup Go
|
|
uses: actions/setup-go@v5
|
|
with:
|
|
go-version: stable
|
|
|
|
- name: install bombardier
|
|
run: go install github.com/codesenberg/bombardier@latest
|
|
|
|
# crow_all.h includes <asio.hpp>, which no runner has out of the box.
|
|
- name: install asio
|
|
if: ${{ inputs.crow }}
|
|
run: |
|
|
if [ "$RUNNER_OS" = "Linux" ]; then
|
|
sudo apt-get update && sudo apt-get install -y libasio-dev
|
|
else
|
|
brew install asio
|
|
fi
|
|
|
|
- name: run benchmark
|
|
run: |
|
|
# Without pipefail the `tee` below swallows a build failure and the
|
|
# job reports success having measured nothing.
|
|
set -o pipefail
|
|
export PATH="$(go env GOPATH)/bin:$PATH"
|
|
CROW_FLAGS="-std=c++17"
|
|
if [ "$RUNNER_OS" = "macOS" ]; then
|
|
CROW_FLAGS="$CROW_FLAGS -I$(brew --prefix asio)/include"
|
|
fi
|
|
if [ "${{ inputs.crow }}" = "true" ]; then TARGET=bench-all; else TARGET=bench; fi
|
|
make -C benchmark "$TARGET" \
|
|
CROW_CXXFLAGS="$CROW_FLAGS" \
|
|
BENCH="bombardier -c ${{ inputs.connections }} -d ${{ inputs.duration }} localhost:8080" \
|
|
2>&1 | tee /tmp/bench.txt
|
|
|
|
# pipefail only catches a failed build. Each Makefile recipe ends in
|
|
# `kill`, so a bombardier that never ran still leaves make happy — check
|
|
# that the measurements are actually there.
|
|
- name: check results were produced
|
|
run: |
|
|
expected=1
|
|
if [ "${{ inputs.crow }}" = "true" ]; then expected=2; fi
|
|
got=$(grep -c "Reqs/sec" /tmp/bench.txt || true)
|
|
if [ "$got" -lt "$expected" ]; then
|
|
echo "::error::expected $expected benchmark result(s), found $got"
|
|
exit 1
|
|
fi
|
|
|
|
- name: record results
|
|
if: always()
|
|
run: |
|
|
{
|
|
echo "## Benchmark (${{ matrix.os }})"
|
|
echo ""
|
|
echo "- ref: \`${{ github.ref_name }}\` (${{ github.sha }})"
|
|
echo "- connections=${{ inputs.connections }} duration=${{ inputs.duration }}"
|
|
echo ""
|
|
echo '```'
|
|
cat /tmp/bench.txt
|
|
echo '```'
|
|
} >> "$GITHUB_STEP_SUMMARY"
|