63 lines
1.8 KiB
YAML
63 lines
1.8 KiB
YAML
name: benchmark-ab
|
|
|
|
# Manual A/B throughput comparison between two refs.
|
|
#
|
|
# This is a measurement, not a test: it never fails the build on a slow result.
|
|
# Absolute req/s from a shared runner is meaningless on its own, so both refs
|
|
# are built and measured alternately in the same job and only the ratio of the
|
|
# medians is reported, with a permutation test to say whether the difference
|
|
# stands out from the run-to-run noise.
|
|
#
|
|
# Non-SSL and Linux only for now.
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
base:
|
|
description: "Baseline ref"
|
|
required: false
|
|
default: "origin/master"
|
|
head:
|
|
description: "Ref to compare (defaults to the ref this run was started on)"
|
|
required: false
|
|
default: ""
|
|
rounds:
|
|
description: "Measurement rounds per ref (9+ recommended; below 4 the test can never reach significance)"
|
|
required: false
|
|
default: "9"
|
|
duration:
|
|
description: "Load duration per measurement"
|
|
required: false
|
|
default: "5s"
|
|
connections:
|
|
description: "Concurrent connections"
|
|
required: false
|
|
default: "10"
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
ubuntu:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: install bombardier
|
|
run: go install github.com/codesenberg/bombardier@latest
|
|
|
|
- name: run A/B benchmark
|
|
run: |
|
|
export PATH="$(go env GOPATH)/bin:$PATH"
|
|
HEAD_REF="${{ inputs.head }}"
|
|
if [ -z "$HEAD_REF" ]; then HEAD_REF="${{ github.sha }}"; fi
|
|
./benchmark/ab.sh \
|
|
--base "${{ inputs.base }}" \
|
|
--head "$HEAD_REF" \
|
|
--rounds "${{ inputs.rounds }}" \
|
|
--duration "${{ inputs.duration }}" \
|
|
--connections "${{ inputs.connections }}"
|