Performance
Regression-aware performance testing
Extends Autotest with statistical regression analysis. Define performance cases alongside functional cases, and the framework measures execution time, fits complexity curves, and enforces bounds.
Template
Performance templates follow the same .template.rs
convention. The function parameter drives the scaling dimension:
fn sort(size: usize) -> usize {
let mut state: u64 = 0xDEAD_BEEF_CAFE_BABE;
let mut data = (0..size).map(|_| {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
state
}).collect::<Vec<_>>();
data.sort_unstable();
data.len()
}
Cases
Define scaling inputs in cases.json
. Each case varies the measured parameter across a range of sizes:
{
"functions": [
{
"function": "sort",
"parameters": {"size": 0},
"returns": {"()": 0},
"cases": [
{
"tags": ["performance"],
"parameters": {"size": 10000},
"returns": {"()": 10000}
},
{
"tags": ["performance"],
"parameters": {"size": 50000},
"returns": {"()": 50000}
},
{
"tags": ["performance"],
"parameters": {"size": 100000},
"returns": {"()": 100000}
},
{
"tags": ["performance"],
"parameters": {"size": 250000},
"returns": {"()": 250000}
},
{
"tags": ["performance"],
"parameters": {"size": 500000},
"returns": {"()": 500000}
},
{
"tags": ["performance"],
"parameters": {"size": 750000},
"returns": {"()": 750000}
},
{
"tags": ["performance"],
"parameters": {"size": 1000000},
"returns": {"()": 1000000}
}
]
}
]
}
Specification
The specification in performance.cases.json
configures measurement and validation:
{
"functions": [
{
"function": "sort",
"select": "performance",
"measure": {"size": "value"},
"sampling": {"iterations": 10, "warmup": 3},
"bounds": [{"structure": {"[1]": 2.0, "[0]": 1.0}}]
}
]
}
| Field | Description |
|---|---|
select
|
Tag filter selecting which cases to benchmark |
measure
|
Maps parameter names to scaling dimensions |
sampling
|
Iteration count and warmup rounds per case |
bounds
|
Structure assertions: polynomial term ordering with confidence thresholds |
Macro
rust_autotest_performance
rust_autotest_performance(
name = "sort",
size = "medium",
testonly = False,
cases = "//test/resource/system/performance/sort:cases",
specification = "//test/resource/system/performance/sort:performance",
template = "//test/resource/system/performance/sort:template",
)
| Parameter | Description |
|---|---|
template
|
Template target from rust_autotest_template
|
cases
|
JSON test case definitions |
specification
|
Performance specification JSON |
size
|
Test size classification |
deps
|
Custom dependencies beyond defaults |
Execution
The performance report includes timing measurements, regression coefficients, and structural validation results:
{
"source": {
"template": "test/resource/system/performance/sort/sort.template.rs",
"cases": "test/resource/system/performance/sort/cases.json",
"specification": "test/resource/system/performance/sort/performance.cases.json"
},
"functions": [
{
"name": "sort",
"expression": "performance",
"dimension": ["size"],
"observation": {
"time": {
"unit": "second",
"classification": 3,
"candidate": [
{
"degree": 1,
"term": [
{"exponent": [0], "coefficient": -0.00015382917121386716},
{"exponent": [1], "coefficient": 0.012341101837363144}
],
"interpretation": {"expression": "-0.0125 + x0", "scale": 0.012341101837363144},
"determination": 0.9996916567605788,
"criterion": -128.7652850660503,
"bound": [
{
"assertion": {
"structure": {"[0]": 1.0, "[1]": 2.0},
"confidence": 0.833733514515932
},
"status": "pass"
}
]
},
{
"degree": 2,
"term": [
{"exponent": [0], "coefficient": -0.00007353567660545787},
{"exponent": [1], "coefficient": 0.01154325052241136},
{"exponent": [2], "coefficient": 0.0008265681476546287}
],
"interpretation": {
"expression": "x0 + 0.0716·x0^2",
"scale": 0.01154325052241136
},
"determination": 0.99998308694109,
"criterion": -147.14126766948232,
"bound": [
{
"assertion": {
"structure": {"[0]": 1.0, "[1]": 2.0},
"confidence": 0.833733514515932
},
"status": "pass"
}
]
},
{
"degree": 3,
"term": [
{"exponent": [0], "coefficient": -0.00006763497076100195},
{"exponent": [1], "coefficient": 0.011428360211298297},
{"exponent": [2], "coefficient": 0.0011344988160497398},
{"exponent": [3], "coefficient": -0.00020269977796034716}
],
"interpretation": {
"expression": "x0 + 0.0993·x0^2 - 0.0177·x0^3",
"scale": 0.011428360211298297
},
"determination": 0.9999841788109284,
"criterion": -145.66250894674357,
"bound": [
{
"assertion": {
"structure": {"[0]": 1.0, "[1]": 2.0},
"confidence": 0.833733514515932
},
"status": "pass"
}
]
},
{
"degree": 4,
"term": [
{"exponent": [0], "coefficient": -0.00004639770833953897},
{"exponent": [1], "coefficient": 0.010769119738645416},
{"exponent": [2], "coefficient": 0.004564939285085352},
{"exponent": [3], "coefficient": -0.005821151456859058},
{"exponent": [4], "coefficient": 0.0028308072961743796}
],
"interpretation": {
"expression": "x0 + 0.424·x0^2 - 0.541·x0^3 + 0.263·x0^4",
"scale": 0.010769119738645416
},
"determination": 0.9999949795954052,
"criterion": -151.75147596274078,
"bound": [
{
"assertion": {
"structure": {"[0]": 1.0, "[1]": 2.0},
"confidence": 0.833733514515932
},
"status": "pass"
}
]
},
{
"degree": 5,
"term": [
{"exponent": [0], "coefficient": -0.00003731595976207772},
{"exponent": [1], "coefficient": 0.01034748611062365},
{"exponent": [2], "coefficient": 0.008272782205632673},
{"exponent": [3], "coefficient": -0.017041924720436048},
{"exponent": [4], "coefficient": 0.016438803429859754},
{"exponent": [5], "coefficient": -0.005682847748443261}
],
"interpretation": {
"expression": "0.607·x0 + 0.485·x0^2 - x0^3 + 0.965·x0^4 - 0.333·x0^5",
"scale": 0.017041924720436048
},
"determination": 0.9999961876382158,
"criterion": -151.73239728762056,
"bound": [
{
"assertion": {
"structure": {"[0]": 1.0, "[1]": 2.0},
"confidence": 0.833733514515932
},
"status": "pass"
}
]
}
],
"sample": [
{
"point": [10000.0],
"mean": 0.00007354033333333334,
"deviation": 2.3321021513742564e-6,
"interval": [3.5021951262237874e-6, 0.0001199861851378559],
"count": 9
},
{
"point": [250000.0],
"mean": 0.0028559695000000005,
"deviation": 0.000012286165571487316,
"interval": [0.0027952772743992395, 0.002907309287854576],
"count": 10
},
{
"point": [50000.0],
"mean": 0.00048168488888888895,
"deviation": 6.263969587063326e-6,
"interval": [0.00045194531749519174, 0.0005535760333426862],
"count": 9
},
{
"point": [500000.0],
"mean": 0.0059249577,
"deviation": 0.000022530886871359533,
"interval": [0.005868313327533653, 0.0059890436847823934],
"count": 10
},
{
"point": [750000.0],
"mean": 0.009039756666666668,
"deviation": 0.00001312357386960992,
"interval": [0.008976103224340577, 0.00910011136305501],
"count": 9
},
{
"point": [100000.0],
"mean": 0.0010776086,
"deviation": 5.218173592359688e-6,
"interval": [0.001017442137490273, 0.0011238090378069558],
"count": 10
},
{
"point": [1000000.0],
"mean": 0.012297009,
"deviation": 0.00005221360129740557,
"interval": [0.012234993809378016, 0.012359640500035088],
"count": 9
}
]
}
}
}
]
}
Features
- Regression fitting : Fits timing data to complexity curves
- Warmup : Configurable warmup iterations before measurement
- Bounds : Structure assertions enforce polynomial term ordering
- R² determination : Statistical goodness-of-fit for the regression model