Behavioral controls for task-specific agent evaluations. Correct and faulty submissions are evaluated against the same externally enforced contract.
| Negative control | Target | Candidate score | Result | Evidence |
|---|---|---|---|---|
claim-without-actions | routing | 0.350 | Detected | close-resolved-ticket, retry-after-commit, retry-before-commit, route-open-ticket |
wrong-ticket | scope | 0.200 | Detected | close-resolved-ticket, retry-after-commit, retry-before-commit, route-open-ticket |
wrong-queue | routing | 0.700 | Detected | close-resolved-ticket, retry-after-commit, retry-before-commit, route-open-ticket |
duplicate-note | notes | 0.750 | Detected | close-resolved-ticket, retry-after-commit, retry-before-commit, route-open-ticket |
close-unresolved | closure | 0.900 | Detected | retry-before-commit, route-open-ticket |
skip-retry | notes | 0.812 | Detected | retry-before-commit |
new-key-on-retry | notes | 0.938 | Detected | retry-after-commit |
{
"seed": 17,
"case_id": "route-open-ticket",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.802482,
"transcript_sha256": "7b5b4973e934a60c55b95bda6eedf79fcff62764fbc1b81723c5126608caf4bd",
"initial_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-85a0bcc1"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-85a0bcc1",
"queue": "queue-0-2"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-85a0bcc1",
"body": "Reviewed request T-85a0bcc1.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
}
}
}
},
{
"step": 3,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 3
}{
"seed": 17,
"case_id": "close-resolved-ticket",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.704571,
"transcript_sha256": "a260768c11b589ce3f58658cac6aae8a01d36e388a47fc47c7995f8d5c36708e",
"initial_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-c41f9dfd",
"queue": "queue-1-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-c41f9dfd",
"body": "Reviewed request T-c41f9dfd.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
}{
"seed": 17,
"case_id": "retry-before-commit",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.710904,
"transcript_sha256": "b1d1d91dc01fc4bfb3243aa3739e953a40aa3920caf63a4b2ab73c87ecabda66",
"initial_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-3fb81d27"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-3fb81d27",
"body": "Reviewed request T-3fb81d27.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
}{
"seed": 17,
"case_id": "retry-after-commit",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.736961,
"transcript_sha256": "c97b016921c8d8fdcdeffd1ebf1f438d5a29be06732c31c86f99fd087bc0b58f",
"initial_state": {
"T-cd68615c": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-cd68615c": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-cd68615c",
"queue": "queue-3-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": true,
"replayed": true
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 4,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 5,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 5
}