EVALARC / EVALUATION
support-routing
business-tools · task v0.1.0 · INCOMPLETE
Dimension results
Case results
Case evidence
route-open-ticket · seed 17 · passed
{
"seed": 17,
"case_id": "route-open-ticket",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.461764,
"transcript_sha256": "7b5b4973e934a60c55b95bda6eedf79fcff62764fbc1b81723c5126608caf4bd",
"initial_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-85a0bcc1": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
},
"U-6a06e9ab": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-85a0bcc1"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-85a0bcc1",
"queue": "queue-0-2"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-85a0bcc1",
"body": "Reviewed request T-85a0bcc1.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-85a0bcc1": {
"before": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-0-2",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-85a0bcc1."
]
}
}
}
},
{
"step": 3,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 3
}close-resolved-ticket · seed 17 · passed
{
"seed": 17,
"case_id": "close-resolved-ticket",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.480463,
"transcript_sha256": "a260768c11b589ce3f58658cac6aae8a01d36e388a47fc47c7995f8d5c36708e",
"initial_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-c41f9dfd": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"U-b48438b5": {
"category": "billing",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-c41f9dfd",
"queue": "queue-1-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-c41f9dfd",
"body": "Reviewed request T-c41f9dfd.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-c41f9dfd"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-c41f9dfd": {
"before": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
},
"after": {
"category": "billing",
"queue": "queue-1-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-c41f9dfd."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
}retry-before-commit · seed 17 · passed
{
"seed": 17,
"case_id": "retry-before-commit",
"checks": {
"routing": true,
"notes": true,
"closure": true,
"scope": true,
"protocol": true
},
"passed": true,
"status": "passed",
"error": null,
"duration_seconds": 0.458016,
"transcript_sha256": "b1d1d91dc01fc4bfb3243aa3739e953a40aa3920caf63a4b2ab73c87ecabda66",
"initial_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-3fb81d27": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
},
"U-62590992": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-3fb81d27"
}
},
"result": {
"ok": true,
"ticket": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-3fb81d27",
"queue": "queue-2-1"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-3fb81d27",
"body": "Reviewed request T-3fb81d27.",
"request_id": "review"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-3fb81d27": {
"before": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "technical",
"queue": "queue-2-1",
"resolved": false,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-3fb81d27."
]
}
}
}
},
{
"step": 4,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 4
}retry-after-commit · seed 17 · failed
{
"seed": 17,
"case_id": "retry-after-commit",
"checks": {
"routing": true,
"notes": false,
"closure": true,
"scope": true,
"protocol": true
},
"passed": false,
"status": "failed",
"error": null,
"duration_seconds": 0.501206,
"transcript_sha256": "4d1123f9ef2ebc43d80c84e4f2054b3ed62a22db7d6c565f0f7f3a9f28c662b4",
"initial_state": {
"T-cd68615c": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"final_state": {
"T-cd68615c": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c.",
"Reviewed request T-cd68615c."
]
},
"U-51436d1f": {
"category": "account",
"queue": "inbox",
"resolved": false,
"status": "open",
"notes": [
"Unrelated request; preserve this ticket."
]
}
},
"trace": [
{
"step": 0,
"type": "tool_call",
"action": {
"tool": "get_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true,
"ticket": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
},
"status": "ok",
"error_source": null,
"changes": {}
},
{
"step": 1,
"type": "tool_call",
"action": {
"tool": "assign_queue",
"arguments": {
"ticket_id": "T-cd68615c",
"queue": "queue-3-0"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "inbox",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
}
}
}
},
{
"step": 2,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review"
}
},
"result": {
"ok": false,
"error": {
"code": "temporarily_unavailable",
"retryable": true
}
},
"status": "tool_error",
"error_source": "injected",
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 3,
"type": "tool_call",
"action": {
"tool": "add_note",
"arguments": {
"ticket_id": "T-cd68615c",
"body": "Reviewed request T-cd68615c.",
"request_id": "review-retry"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 4,
"type": "tool_call",
"action": {
"tool": "close_ticket",
"arguments": {
"ticket_id": "T-cd68615c"
}
},
"result": {
"ok": true
},
"status": "ok",
"error_source": null,
"changes": {
"T-cd68615c": {
"before": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "open",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c.",
"Reviewed request T-cd68615c."
]
},
"after": {
"category": "billing",
"queue": "queue-3-0",
"resolved": true,
"status": "closed",
"notes": [
"Customer contacted support.",
"Reviewed request T-cd68615c.",
"Reviewed request T-cd68615c."
]
}
}
}
},
{
"step": 5,
"type": "finish",
"action": {
"finish": true
}
}
],
"tool_calls": 5
}Reproduction record
Candidate SHA-256: b89a99819c03e8290096ca268f637c5517d465b3ec5f0b5fe2b21c35509ff202
Grader SHA-256: 2dced2a842369299e6e924db626fedc120071d5731cd9b79202dba7395b90255
Cases SHA-256: b7cd8663c69da9e1715d5c8e1d707514863dcc333fefc57acc3315053ebc85df
Created: 2026-09-14T01:58:21.160793+00:00
Seeds: [17]
Runtime: {"backend": "docker", "image": "python:3.12-slim", "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", "command": ["{python}", "-I", "-B", "main.py"], "response_timeout_seconds": 10.0, "session_output_limit_bytes": 1048576, "python": "3.12.3", "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39"}
Complete JSON evidence