Evals run before every pull request Evals used to be opt-in. Now every diff runs your tests and Kestrel's evals for scope, style and risk before a pull request can open, and a failing diff goes straight back to the agent with the failure attached.
Eval gate
An agent writes a diff that runs through six checks in order; it fails the evals and goes back to the agent with the failure, and the revised diff passes every check before a pull request opens.
0.5 ? 1.1 : kind < -0.5 ? 0.5 : 0.2;
if (c > len + 0.5) return 0.0;
return kind > 0.5 ? 0.68 : kind < -0.5 ? 0.4 : 0.3;
}
// The diff at centre q, written as far as w (0 to 1, line by line), with its failure attached under it (fl).
float diff(vec2 p, vec2 q, float ver, float w, float fl, float k) {
float c = floor((p.x - q.x) / 7.0 + 2.5);
float r = floor((p.y - q.y) / 9.4 + 2.5);
if (c < 0.0 || c > 4.0 || r < 0.0 || r > 5.0) return 0.0;
if (r > 4.5) return fl * (c < 0.5 ? 1.3 : 1.05);
if (r + c / 5.0 >= w * 5.0) return 0.0;
return k * hunk(c, r, ver);
}
// The point a distance d along the return path.
vec2 back(float d) {
if (d <= 21.0) return mix(vec2(427.5, 200.9), vec2(406.5, 200.9), (d - 0.0) / 21.0);
if (d <= 124.4) return mix(vec2(406.5, 200.9), vec2(406.5, 304.3), (d - 21.0) / 103.4);
if (d <= 421.9) return mix(vec2(406.5, 304.3), vec2(109.0, 304.3), (d - 124.4) / 297.5);
if (d <= 489.3) return mix(vec2(109.0, 304.3), vec2(109.0, 236.9), (d - 421.9) / 67.4);
return vec2(109.0, 236.9);
}
// One check's run for a diff that reached it at a: its cells light out from the lane, a cell at a time, up and down
// the stage; stopAt under 7 is a test that failed there, and the run stops. Then the light fades back to idle.
float scan(float dc, float a, float stopAt, float t) {
float s0 = a - 0.06;
if (t < s0) return 0.0;
float reach = min(7.0 * (t - s0) / 0.12, stopAt);
if (dc > floor(reach) + 0.5) return 0.0;
float done = s0 + 0.12 * stopAt / 7.0;
float fade = 1.0 - sm(done, done + (stopAt < 6.5 ? 0.25 : 0.45), t);
float head = (t < done && abs(dc - floor(reach)) < 0.5) ? 0.2 : 0.0;
return fade * (1.0 + head);
}
// Held across the seam: on from on0, and still on from the last cycle until the board clears.
float held(float on0, float t) { return max(sm(on0, on0 + 0.06, t), 1.0 - sm(0.25, 0.43, t)); }
float field(vec2 p, float t) {
if (p.x < 18.0 || p.x > 702.0) return 0.0;
float ph = TAU * t / T;
float col = floor((p.x - 4.0) / 7.0 + 0.5) - 2.0;
float cell = floor(p.y / 9.4);
float v = 0.0;
// the agent: a bright rim, an inner ring, a speckled floor and a core that churns, and flares while it writes
float wr = sm(0.2, 0.3, t) * (1.0 - sm(0.5, 0.7, t))
+ sm(2.26, 2.36, t) * (1.0 - sm(2.58, 2.78, t));
{
vec2 c = vec2(109.0, LANE);
float d = length(p - c) / 30.0;
float inside = 1.0 - sm(0.9, 1.0, d);
float churn = noise(p * 0.09 + vec2(1.4 * cos(ph), 1.4 * sin(ph)));
v += 0.46 * exp(-pow(d - 1.0, 2.0) / 0.008);
v += inside * 0.26 * exp(-pow(d - 0.64, 2.0) / 0.005);
v += inside * (0.12 + 0.12 * step(0.82, hash(floor(p / vec2(7.0, 9.4)) + 1.3)));
v += blob(p, c, 13.2, 0.82 + 0.5 * (churn - 0.5) + 0.45 * wr);
}
// the forward lane, agent to pull request, and the return lane under the gate, back up into the agent
if (abs(cell - LK) < 0.5 && col > 17.0 && col < 81.0) v = max(v, 0.2);
if (abs(cell - 32.0) < 0.5 && col >= 13.0 && col <= 65.0) v = max(v, 0.2);
if (abs(col - 13.0) < 0.5 && cell > 24.0 && cell < 32.0) v = max(v, 0.2);
// the gate's floor: a speckle between the stages, so the gate reads as one housing
if (col > 32.0 && col < 67.0 && abs(cell - LK) < 7.5 && step(0.62, hash(vec2(col, cell) + 2.3)) > 0.5) v = max(v, 0.22);
// the gate: six stages of test cells, a header and footer cell on each, in order; each runs as the diff reaches it
float rel = col - 34.0;
float s = floor(rel / 6.0);
float dc = abs(cell - LK);
if (s >= 0.0 && s <= 5.0 && rel - 6.0 * s < 1.5 && dc < 7.5) {
float hs = hash(vec2(col, cell) + 7.1);
float base = dc > 6.5 ? 0.85 : 0.42 + 0.2 * step(0.55, hs) * (0.7 + 0.3 * sin(ph + hs * 40.0));
float a1 = 0.84 + s * 0.18;
float a2 = 2.92 + s * 0.18;
float lit = 0.0;
if (s < 4.0 - 0.5) lit = scan(dc, a1, 7.0, t);
else if (s < 4.0 + 0.5) lit = scan(dc, a1, 5.0, t);
lit = max(lit, scan(dc, a2, 7.0, t));
v = max(v, max(base, lit * (dc > 6.5 ? 1.1 : 1.0)));
// the failing test: it flares as the run reaches it, and dims once the diff is back at the agent
if (s > 4.0 - 0.5 && s < 4.0 + 0.5 && abs(cell - 16.0) < 0.5)
v = max(v, 1.3 * sm(1.576, 1.616, t) * (1.0 - sm(2.07, 2.42, t)));
}
// the record: a row of six slots per run, each filled as its check passes; run 1 stops at EVALS
if (s >= 0.0 && s <= 5.0 && rel - 6.0 * s < 1.5) {
if (cell > 35.5 && cell < 37.5) {
float fl = s < 4.0 - 0.5 ? held(0.9 + s * 0.18, t) : 0.0;
v = max(v, 0.15 + 0.85 * fl);
}
if (cell > 38.5 && cell < 40.5) v = max(v, 0.15 + 0.85 * held(2.98 + s * 0.18, t));
}
// the pull request: a faint outline waiting; it opens once the diff that passed reaches it
float open = held(4.18, t);
float pc = col - 81.0;
float pk = cell - 16.0;
if (pc >= 0.0 && pc <= 8.0 && pk >= 0.0 && pk <= 10.0) {
bool rim = pc < 0.5 || pc > 7.5 || pk < 0.5 || pk > 9.5;
float card = rim ? 0.42 : 0.0;
if (abs(pk - 1.0) < 0.5 && pc >= 1.0 && pc <= 6.0) card = 1.05; // the title
if (abs(pk - 2.0) < 0.5 && pc >= 1.0 && pc <= 3.0) card = 0.3; // the branch
if (pk > 3.5 && pk < 8.5 && pc >= 1.0 && pc <= 5.0) card = max(card, 0.95 * hunk(pc - 1.0, pk - 4.0, 1.0)); // the diff's lines
if (abs(pk - 9.0) < 0.5 && pc >= 1.0 && pc <= 6.0) card = max(0.2, 0.95 * held(4.24 + 0.035 * pc, t)); // six check pips
v = max(v, rim ? max(0.15, card * open) : card * open);
}
// diff 1: written, through the four checks it passes, stopped at EVALS, then back to the agent with its failure
float fl = sm(1.586, 1.646, t);
if (t > 0.25 && t < 0.5) v = max(v, diff(p, vec2(179.0, LANE), 0.0, (t - 0.25) / 0.25, 0.0, 1.0));
else if (t >= 0.5 && t < 0.84) v = max(v, diff(p, vec2(mix(179.0, 259.5, ease((t - 0.5) / 0.34)), LANE), 0.0, 1.0, 0.0, 1.0));
else if (t >= 0.84 && t < 1.66) {
float x = 259.5 + 42.0 * stepx(min((t - 0.84) / 0.18, 4.0));
v = max(v, diff(p, vec2(x, LANE), 0.0, 1.0, fl, 1.0));
} else if (t >= 1.66 && t < 2.32) {
float u = (t - 1.66) / 0.66;
float k = 1.0 - sm(0.88, 1.0, u);
v = max(v, diff(p, back(ease(u) * 489.3), 0.0, 1.0, k, k));
}
// diff 2: written, through all six checks, out to the pull request
if (t > 2.34 && t < 2.58) v = max(v, diff(p, vec2(179.0, LANE), 1.0, (t - 2.34) / 0.24, 0.0, 1.0));
else if (t >= 2.58 && t < 2.92) v = max(v, diff(p, vec2(mix(179.0, 259.5, ease((t - 2.58) / 0.34)), LANE), 1.0, 1.0, 0.0, 1.0));
else if (t >= 2.92 && t < 3.88) v = max(v, diff(p, vec2(259.5 + 42.0 * stepx((t - 2.92) / 0.18), LANE), 1.0, 1.0, 0.0, 1.0));
else if (t >= 3.88 && t < 4.2) {
float u = (t - 3.88) / 0.32;
v = max(v, diff(p, vec2(mix(469.5, 613.0, ease(u)), LANE), 1.0, 1.0, 0.0, 1.0 - sm(0.8, 1.0, u)));
}
return v;
}]]>
Eval results show on the pull request as a check Write your own evals in kestrel.yaml Failed evals no longer count as a new run