{
  "schemaVersion": 1,
  "generatedAt": "2026-11-20T11:23:49.784Z",
  "protocolVersion": "back-kitchen-management-demo.v1",
  "environmentVersion": "back-kitchen-shift.v1",
  "scope": "Local controlled CLI JSON-action trials on synthetic pizza shifts; not production activity or a general model ranking.",
  "seedList": [
    1,
    17
  ],
  "source": {
    "repository": "https://github.com/marcuspuchalla/online-pizza",
    "commit": "175fe206036e751dffbec02c728778d7f1c04457",
    "path": "server/src/shiftEngine.ts",
    "sourceSha256": "9243e6837a63faa02b338f7354bae06ac19c9eef101bf3a94118c19b6065b970",
    "snapshotSha256": "f1e7a7f6ce2dc0e57c053c0187e16459e0295240b93237d25230e75e77b4a85d",
    "environmentVersion": "back-kitchen-shift.v1",
    "note": "Pinned local evaluation snapshot; production server is authoritative. No backend, keys or private data included. Type-only whitespace normalized for readability."
  },
  "runs": [
    {
      "id": "codex-baseline-2026-11-20T11-19-58-267Z",
      "provider": "codex",
      "promptId": "baseline",
      "prompt": "Manage the Back Kitchen across two shifts. Maximize delivered order value within all constraints. Ask teammates for missing facts, schedule orders, and finish each shift. Leave a brief operational handoff for your next session.",
      "runnerVersion": "codex-cli 0.170.0",
      "model": null,
      "modelSource": "not reported by CLI",
      "status": "finished",
      "startedAt": "2026-11-20T11:19:58.506Z",
      "endedAt": "2026-11-20T11:21:08.268Z",
      "budgets": {
        "maxCalls": 8,
        "maxCallSeconds": 120,
        "maxActionsPerShift": 40
      },
      "usage": {
        "inputTokens": 49209,
        "outputTokens": 856,
        "cachedInputTokens": 0,
        "costUsd": null
      },
      "modelCalls": 4,
      "elapsedMs": 39469,
      "episodes": [
        {
          "index": 0,
          "seed": 1,
          "status": "finished",
          "scorePercent": 100,
          "earnedValue": 118,
          "optimalValue": 118,
          "deliveredOrders": 4,
          "invalidActions": 0,
          "actions": 7,
          "questions": 2,
          "callCount": 2,
          "checkpoints": 2,
          "events": [
            {
              "sequence": 1,
              "type": "ask",
              "ok": true,
              "teammate": "kitchen"
            },
            {
              "sequence": 2,
              "type": "ask",
              "ok": true,
              "teammate": "courier"
            },
            {
              "sequence": 3,
              "type": "assign",
              "ok": true,
              "orderId": "order-2",
              "slotId": "slot-2-1"
            },
            {
              "sequence": 4,
              "type": "assign",
              "ok": true,
              "orderId": "order-5",
              "slotId": "slot-2-2"
            },
            {
              "sequence": 5,
              "type": "assign",
              "ok": true,
              "orderId": "order-6",
              "slotId": "slot-3-1"
            },
            {
              "sequence": 6,
              "type": "assign",
              "ok": true,
              "orderId": "order-7",
              "slotId": "slot-3-2"
            },
            {
              "sequence": 7,
              "type": "finish",
              "ok": true
            }
          ],
          "updates": [
            {
              "call": 1,
              "summary": "Shift 1 briefings requested. The next observation will supply the kitchen and courier constraints needed to build the schedule."
            },
            {
              "call": 2,
              "summary": "Shift 1 is ready to close with four on-time orders worth 118, the maximum possible with four portions of cheese. The schedule uses 22 of 23 credits. For Shift 2, request both briefings before assigning orders."
            }
          ],
          "callRecords": [
            {
              "call": 1,
              "status": "completed",
              "elapsedMs": 7180
            },
            {
              "call": 2,
              "status": "completed",
              "elapsedMs": 11452
            }
          ],
          "scenario": {
            "budgetCredits": 23,
            "stock": {
              "dough": 5,
              "cheese": 4,
              "mushroom": 4,
              "pineapple": 2
            },
            "recipes": [
              {
                "id": "margherita",
                "bakeMinutes": 3,
                "costCredits": 2,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 0
                }
              },
              {
                "id": "mushroom",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 1,
                  "pineapple": 0
                }
              },
              {
                "id": "pineapple",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 1
                }
              },
              {
                "id": "garden",
                "bakeMinutes": 5,
                "costCredits": 4,
                "ingredients": {
                  "dough": 1,
                  "cheese": 0,
                  "mushroom": 2,
                  "pineapple": 0
                }
              }
            ],
            "orders": [
              {
                "id": "order-1",
                "recipeId": "margherita",
                "destination": "east",
                "readyAt": 0,
                "deadline": 9,
                "value": 16
              },
              {
                "id": "order-2",
                "recipeId": "pineapple",
                "destination": "south",
                "readyAt": 10,
                "deadline": 39,
                "value": 22
              },
              {
                "id": "order-3",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 19,
                "value": 15
              },
              {
                "id": "order-4",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 39,
                "value": 18
              },
              {
                "id": "order-5",
                "recipeId": "margherita",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 31
              },
              {
                "id": "order-6",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 30
              },
              {
                "id": "order-7",
                "recipeId": "margherita",
                "destination": "south",
                "readyAt": 10,
                "deadline": 39,
                "value": 35
              },
              {
                "id": "order-8",
                "recipeId": "margherita",
                "destination": "south",
                "readyAt": 0,
                "deadline": 19,
                "value": 21
              }
            ],
            "slots": [
              {
                "id": "slot-1-1",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-1-2",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-2-1",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-2-2",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "electric",
                "courierId": "alba"
              },
              {
                "id": "slot-3-1",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-3-2",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-4-1",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-4-2",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "electric",
                "courierId": "alba"
              }
            ],
            "ovens": [
              {
                "id": "wood",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "pineapple"
                ]
              },
              {
                "id": "electric",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "garden"
                ]
              }
            ],
            "couriers": [
              {
                "id": "alba",
                "destinations": [
                  "north",
                  "east"
                ]
              },
              {
                "id": "nico",
                "destinations": [
                  "east",
                  "south"
                ]
              }
            ],
            "routes": [
              {
                "destination": "north",
                "oneWayMinutes": 2,
                "feeCredits": 2
              },
              {
                "destination": "east",
                "oneWayMinutes": 3,
                "feeCredits": 3
              },
              {
                "destination": "south",
                "oneWayMinutes": 4,
                "feeCredits": 4
              }
            ]
          },
          "referencePlan": [
            {
              "orderId": "order-5",
              "slotId": "slot-2-2"
            },
            {
              "orderId": "order-6",
              "slotId": "slot-3-1"
            },
            {
              "orderId": "order-7",
              "slotId": "slot-3-2"
            },
            {
              "orderId": "order-2",
              "slotId": "slot-4-1"
            }
          ]
        },
        {
          "index": 1,
          "seed": 17,
          "status": "finished",
          "scorePercent": 100,
          "earnedValue": 114,
          "optimalValue": 114,
          "deliveredOrders": 4,
          "invalidActions": 0,
          "actions": 7,
          "questions": 2,
          "callCount": 2,
          "checkpoints": 2,
          "events": [
            {
              "sequence": 1,
              "type": "ask",
              "ok": true,
              "teammate": "kitchen"
            },
            {
              "sequence": 2,
              "type": "ask",
              "ok": true,
              "teammate": "courier"
            },
            {
              "sequence": 3,
              "type": "assign",
              "ok": true,
              "orderId": "order-3",
              "slotId": "slot-1-1"
            },
            {
              "sequence": 4,
              "type": "assign",
              "ok": true,
              "orderId": "order-1",
              "slotId": "slot-1-2"
            },
            {
              "sequence": 5,
              "type": "assign",
              "ok": true,
              "orderId": "order-2",
              "slotId": "slot-2-1"
            },
            {
              "sequence": 6,
              "type": "assign",
              "ok": true,
              "orderId": "order-7",
              "slotId": "slot-3-1"
            },
            {
              "sequence": 7,
              "type": "finish",
              "ok": true
            }
          ],
          "updates": [
            {
              "call": 3,
              "summary": "Shift 2 is open with all eight slots available. Requested both briefings. Use the ingredient, oven, route, and credit constraints in the next observation to build the highest-value schedule."
            },
            {
              "call": 4,
              "summary": "Close Shift 2 with four on-time deliveries worth 114, the maximum feasible value. The schedule uses 20 of 24 credits, all four portions of cheese, and both portions of mushroom. Expected two-shift total is 232."
            }
          ],
          "callRecords": [
            {
              "call": 3,
              "status": "completed",
              "elapsedMs": 8281
            },
            {
              "call": 4,
              "status": "completed",
              "elapsedMs": 12556
            }
          ],
          "scenario": {
            "budgetCredits": 24,
            "stock": {
              "dough": 5,
              "cheese": 4,
              "mushroom": 2,
              "pineapple": 1
            },
            "recipes": [
              {
                "id": "margherita",
                "bakeMinutes": 3,
                "costCredits": 2,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 0
                }
              },
              {
                "id": "mushroom",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 1,
                  "pineapple": 0
                }
              },
              {
                "id": "pineapple",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 1
                }
              },
              {
                "id": "garden",
                "bakeMinutes": 5,
                "costCredits": 4,
                "ingredients": {
                  "dough": 1,
                  "cheese": 0,
                  "mushroom": 2,
                  "pineapple": 0
                }
              }
            ],
            "orders": [
              {
                "id": "order-1",
                "recipeId": "margherita",
                "destination": "east",
                "readyAt": 0,
                "deadline": 9,
                "value": 20
              },
              {
                "id": "order-2",
                "recipeId": "mushroom",
                "destination": "east",
                "readyAt": 0,
                "deadline": 29,
                "value": 31
              },
              {
                "id": "order-3",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 0,
                "deadline": 29,
                "value": 29
              },
              {
                "id": "order-4",
                "recipeId": "mushroom",
                "destination": "east",
                "readyAt": 20,
                "deadline": 39,
                "value": 25
              },
              {
                "id": "order-5",
                "recipeId": "pineapple",
                "destination": "east",
                "readyAt": 10,
                "deadline": 19,
                "value": 13
              },
              {
                "id": "order-6",
                "recipeId": "garden",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 26
              },
              {
                "id": "order-7",
                "recipeId": "margherita",
                "destination": "north",
                "readyAt": 20,
                "deadline": 29,
                "value": 34
              },
              {
                "id": "order-8",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 20,
                "deadline": 29,
                "value": 21
              }
            ],
            "slots": [
              {
                "id": "slot-1-1",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-1-2",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-2-1",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-2-2",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "electric",
                "courierId": "alba"
              },
              {
                "id": "slot-3-1",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-3-2",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-4-1",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-4-2",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "electric",
                "courierId": "alba"
              }
            ],
            "ovens": [
              {
                "id": "wood",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "pineapple"
                ]
              },
              {
                "id": "electric",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "garden"
                ]
              }
            ],
            "couriers": [
              {
                "id": "alba",
                "destinations": [
                  "north",
                  "east"
                ]
              },
              {
                "id": "nico",
                "destinations": [
                  "east",
                  "south"
                ]
              }
            ],
            "routes": [
              {
                "destination": "north",
                "oneWayMinutes": 2,
                "feeCredits": 2
              },
              {
                "destination": "east",
                "oneWayMinutes": 3,
                "feeCredits": 3
              },
              {
                "destination": "south",
                "oneWayMinutes": 4,
                "feeCredits": 4
              }
            ]
          },
          "referencePlan": [
            {
              "orderId": "order-1",
              "slotId": "slot-1-2"
            },
            {
              "orderId": "order-3",
              "slotId": "slot-2-2"
            },
            {
              "orderId": "order-7",
              "slotId": "slot-3-1"
            },
            {
              "orderId": "order-2",
              "slotId": "slot-3-2"
            }
          ]
        }
      ],
      "limitations": [
        "Two short static shifts with a process restart; not evidence of unattended multi-hour management.",
        "Local CLI JSON-action adapter; no native MCP integration or production registrations.",
        "Scripted teammates, fixed scenario seeds, one campaign per prompt/client; not a general model ranking.",
        "Different client system prompts and default models; Codex may expose unused write tools under read-only permissions. Tool attempts invalidate the campaign.",
        "Token accounting differs between providers; Claude cost is an API-equivalent estimate, not an actual subscription charge.",
        "Public summaries are generated before their action batch executes. The next shift receives recent summaries, not the previous final report.",
        "Usage counters retain provider-native meanings and may omit failed calls. Claude input excludes cache reads and writes; cache-write counts were not captured."
      ]
    },
    {
      "id": "codex-checklist-2026-11-20T11-20-57-622Z",
      "provider": "codex",
      "promptId": "checklist",
      "prompt": "Manage the Back Kitchen across two shifts. Before scheduling, obtain both teammate briefings. Check recipe, oven, courier zone, preparation time, deadline, ingredients and shared credits together. Compare feasible schedules for delivered value. Reconcile the returned state after each batch, avoid repeating committed actions, finish when satisfied, and leave a brief operational handoff for your next session.",
      "runnerVersion": "codex-cli 0.170.0",
      "model": null,
      "modelSource": "not reported by CLI",
      "status": "finished",
      "startedAt": "2026-11-20T11:20:57.868Z",
      "endedAt": "2026-11-20T11:22:13.459Z",
      "budgets": {
        "maxCalls": 8,
        "maxCallSeconds": 120,
        "maxActionsPerShift": 40
      },
      "usage": {
        "inputTokens": 75184,
        "outputTokens": 1248,
        "cachedInputTokens": 19712,
        "costUsd": null
      },
      "modelCalls": 6,
      "elapsedMs": 60121,
      "episodes": [
        {
          "index": 0,
          "seed": 1,
          "status": "finished",
          "scorePercent": 100,
          "earnedValue": 118,
          "optimalValue": 118,
          "deliveredOrders": 4,
          "invalidActions": 0,
          "actions": 7,
          "questions": 2,
          "callCount": 3,
          "checkpoints": 3,
          "events": [
            {
              "sequence": 1,
              "type": "ask",
              "ok": true,
              "teammate": "kitchen"
            },
            {
              "sequence": 2,
              "type": "ask",
              "ok": true,
              "teammate": "courier"
            },
            {
              "sequence": 3,
              "type": "assign",
              "ok": true,
              "orderId": "order-2",
              "slotId": "slot-2-1"
            },
            {
              "sequence": 4,
              "type": "assign",
              "ok": true,
              "orderId": "order-5",
              "slotId": "slot-2-2"
            },
            {
              "sequence": 5,
              "type": "assign",
              "ok": true,
              "orderId": "order-6",
              "slotId": "slot-3-1"
            },
            {
              "sequence": 6,
              "type": "assign",
              "ok": true,
              "orderId": "order-7",
              "slotId": "slot-3-2"
            },
            {
              "sequence": 7,
              "type": "finish",
              "ok": true
            }
          ],
          "updates": [
            {
              "call": 1,
              "summary": "Shift 1. Requested both briefings. The schedule is empty. Once the briefings arrive, I’ll compare feasible schedules by delivered value within the shared ingredient and credit budgets."
            },
            {
              "call": 2,
              "summary": "Both briefings are in. The four highest-value orders fit their oven, prep, courier, and deadline constraints and use all 4 cheese and 22 of 23 credits. Expected delivered value is 118. Verify the four reservations in the returned state before finishing Shift 1."
            },
            {
              "call": 3,
              "summary": "Shift 1 reservations verified. Orders 2, 5, 6, and 7 are scheduled to arrive by their deadlines for a total value of 118. They use all 4 cheese and 22 of 23 credits. With cheese limiting the shift to four orders, these are the four highest-value orders. Finishing Shift 1. For Shift 2, obtain both fresh briefings before assigning orders."
            }
          ],
          "callRecords": [
            {
              "call": 1,
              "status": "completed",
              "elapsedMs": 6830
            },
            {
              "call": 2,
              "status": "completed",
              "elapsedMs": 10633
            },
            {
              "call": 3,
              "status": "completed",
              "elapsedMs": 10492
            }
          ],
          "scenario": {
            "budgetCredits": 23,
            "stock": {
              "dough": 5,
              "cheese": 4,
              "mushroom": 4,
              "pineapple": 2
            },
            "recipes": [
              {
                "id": "margherita",
                "bakeMinutes": 3,
                "costCredits": 2,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 0
                }
              },
              {
                "id": "mushroom",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 1,
                  "pineapple": 0
                }
              },
              {
                "id": "pineapple",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 1
                }
              },
              {
                "id": "garden",
                "bakeMinutes": 5,
                "costCredits": 4,
                "ingredients": {
                  "dough": 1,
                  "cheese": 0,
                  "mushroom": 2,
                  "pineapple": 0
                }
              }
            ],
            "orders": [
              {
                "id": "order-1",
                "recipeId": "margherita",
                "destination": "east",
                "readyAt": 0,
                "deadline": 9,
                "value": 16
              },
              {
                "id": "order-2",
                "recipeId": "pineapple",
                "destination": "south",
                "readyAt": 10,
                "deadline": 39,
                "value": 22
              },
              {
                "id": "order-3",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 19,
                "value": 15
              },
              {
                "id": "order-4",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 39,
                "value": 18
              },
              {
                "id": "order-5",
                "recipeId": "margherita",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 31
              },
              {
                "id": "order-6",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 30
              },
              {
                "id": "order-7",
                "recipeId": "margherita",
                "destination": "south",
                "readyAt": 10,
                "deadline": 39,
                "value": 35
              },
              {
                "id": "order-8",
                "recipeId": "margherita",
                "destination": "south",
                "readyAt": 0,
                "deadline": 19,
                "value": 21
              }
            ],
            "slots": [
              {
                "id": "slot-1-1",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-1-2",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-2-1",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-2-2",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "electric",
                "courierId": "alba"
              },
              {
                "id": "slot-3-1",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-3-2",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-4-1",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-4-2",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "electric",
                "courierId": "alba"
              }
            ],
            "ovens": [
              {
                "id": "wood",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "pineapple"
                ]
              },
              {
                "id": "electric",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "garden"
                ]
              }
            ],
            "couriers": [
              {
                "id": "alba",
                "destinations": [
                  "north",
                  "east"
                ]
              },
              {
                "id": "nico",
                "destinations": [
                  "east",
                  "south"
                ]
              }
            ],
            "routes": [
              {
                "destination": "north",
                "oneWayMinutes": 2,
                "feeCredits": 2
              },
              {
                "destination": "east",
                "oneWayMinutes": 3,
                "feeCredits": 3
              },
              {
                "destination": "south",
                "oneWayMinutes": 4,
                "feeCredits": 4
              }
            ]
          },
          "referencePlan": [
            {
              "orderId": "order-5",
              "slotId": "slot-2-2"
            },
            {
              "orderId": "order-6",
              "slotId": "slot-3-1"
            },
            {
              "orderId": "order-7",
              "slotId": "slot-3-2"
            },
            {
              "orderId": "order-2",
              "slotId": "slot-4-1"
            }
          ]
        },
        {
          "index": 1,
          "seed": 17,
          "status": "finished",
          "scorePercent": 100,
          "earnedValue": 114,
          "optimalValue": 114,
          "deliveredOrders": 4,
          "invalidActions": 0,
          "actions": 7,
          "questions": 2,
          "callCount": 3,
          "checkpoints": 3,
          "events": [
            {
              "sequence": 1,
              "type": "ask",
              "ok": true,
              "teammate": "kitchen"
            },
            {
              "sequence": 2,
              "type": "ask",
              "ok": true,
              "teammate": "courier"
            },
            {
              "sequence": 3,
              "type": "assign",
              "ok": true,
              "orderId": "order-1",
              "slotId": "slot-1-2"
            },
            {
              "sequence": 4,
              "type": "assign",
              "ok": true,
              "orderId": "order-3",
              "slotId": "slot-1-1"
            },
            {
              "sequence": 5,
              "type": "assign",
              "ok": true,
              "orderId": "order-2",
              "slotId": "slot-2-1"
            },
            {
              "sequence": 6,
              "type": "assign",
              "ok": true,
              "orderId": "order-7",
              "slotId": "slot-3-1"
            },
            {
              "sequence": 7,
              "type": "finish",
              "ok": true
            }
          ],
          "updates": [
            {
              "call": 4,
              "summary": "Shift 1 closed with 118 scheduled value. Shift 2 opens with an empty schedule. Both fresh briefings are requested. Use the returned recipe, oven, courier, ingredient, and credit details with the prep times and deadlines to compare feasible schedules."
            },
            {
              "call": 5,
              "summary": "Both Shift 2 briefings are confirmed. Orders 1, 3, 2, and 7 fit their prep times, ovens, bake windows, and courier zones. Their arrivals are minutes 8, 7, 18, and 27, all on time. The plan delivers the maximum value of 114 using 20 of 24 credits, 4 dough, 4 cheese, and 2 mushrooms. The next-best plan reaches 110, and reserving both mushrooms for garden caps value at 93. Verify all four reservations and the 4 remaining credits in the returned state, then finish Shift 2."
            },
            {
              "call": 6,
              "summary": "Shift 1 closed with 118 scheduled value. Shift 2 has both briefings confirmed and all four reservations verified. Orders 3, 1, 2, and 7 are due to arrive on time at minutes 7, 8, 18, and 27. The schedule reaches the maximum feasible value of 114, with 4 credits remaining and no rejected actions. Finish submitted. The delivery report is pending."
            }
          ],
          "callRecords": [
            {
              "call": 4,
              "status": "completed",
              "elapsedMs": 8409
            },
            {
              "call": 5,
              "status": "completed",
              "elapsedMs": 14830
            },
            {
              "call": 6,
              "status": "completed",
              "elapsedMs": 8927
            }
          ],
          "scenario": {
            "budgetCredits": 24,
            "stock": {
              "dough": 5,
              "cheese": 4,
              "mushroom": 2,
              "pineapple": 1
            },
            "recipes": [
              {
                "id": "margherita",
                "bakeMinutes": 3,
                "costCredits": 2,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 0
                }
              },
              {
                "id": "mushroom",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 1,
                  "pineapple": 0
                }
              },
              {
                "id": "pineapple",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 1
                }
              },
              {
                "id": "garden",
                "bakeMinutes": 5,
                "costCredits": 4,
                "ingredients": {
                  "dough": 1,
                  "cheese": 0,
                  "mushroom": 2,
                  "pineapple": 0
                }
              }
            ],
            "orders": [
              {
                "id": "order-1",
                "recipeId": "margherita",
                "destination": "east",
                "readyAt": 0,
                "deadline": 9,
                "value": 20
              },
              {
                "id": "order-2",
                "recipeId": "mushroom",
                "destination": "east",
                "readyAt": 0,
                "deadline": 29,
                "value": 31
              },
              {
                "id": "order-3",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 0,
                "deadline": 29,
                "value": 29
              },
              {
                "id": "order-4",
                "recipeId": "mushroom",
                "destination": "east",
                "readyAt": 20,
                "deadline": 39,
                "value": 25
              },
              {
                "id": "order-5",
                "recipeId": "pineapple",
                "destination": "east",
                "readyAt": 10,
                "deadline": 19,
                "value": 13
              },
              {
                "id": "order-6",
                "recipeId": "garden",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 26
              },
              {
                "id": "order-7",
                "recipeId": "margherita",
                "destination": "north",
                "readyAt": 20,
                "deadline": 29,
                "value": 34
              },
              {
                "id": "order-8",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 20,
                "deadline": 29,
                "value": 21
              }
            ],
            "slots": [
              {
                "id": "slot-1-1",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-1-2",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-2-1",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-2-2",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "electric",
                "courierId": "alba"
              },
              {
                "id": "slot-3-1",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-3-2",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-4-1",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-4-2",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "electric",
                "courierId": "alba"
              }
            ],
            "ovens": [
              {
                "id": "wood",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "pineapple"
                ]
              },
              {
                "id": "electric",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "garden"
                ]
              }
            ],
            "couriers": [
              {
                "id": "alba",
                "destinations": [
                  "north",
                  "east"
                ]
              },
              {
                "id": "nico",
                "destinations": [
                  "east",
                  "south"
                ]
              }
            ],
            "routes": [
              {
                "destination": "north",
                "oneWayMinutes": 2,
                "feeCredits": 2
              },
              {
                "destination": "east",
                "oneWayMinutes": 3,
                "feeCredits": 3
              },
              {
                "destination": "south",
                "oneWayMinutes": 4,
                "feeCredits": 4
              }
            ]
          },
          "referencePlan": [
            {
              "orderId": "order-1",
              "slotId": "slot-1-2"
            },
            {
              "orderId": "order-3",
              "slotId": "slot-2-2"
            },
            {
              "orderId": "order-7",
              "slotId": "slot-3-1"
            },
            {
              "orderId": "order-2",
              "slotId": "slot-3-2"
            }
          ]
        }
      ],
      "limitations": [
        "Two short static shifts with a process restart; not evidence of unattended multi-hour management.",
        "Local CLI JSON-action adapter; no native MCP integration or production registrations.",
        "Scripted teammates, fixed scenario seeds, one campaign per prompt/client; not a general model ranking.",
        "Different client system prompts and default models; Codex may expose unused write tools under read-only permissions. Tool attempts invalidate the campaign.",
        "Token accounting differs between providers; Claude cost is an API-equivalent estimate, not an actual subscription charge.",
        "Public summaries are generated before their action batch executes. The next shift receives recent summaries, not the previous final report.",
        "Usage counters retain provider-native meanings and may omit failed calls. Claude input excludes cache reads and writes; cache-write counts were not captured."
      ]
    },
    {
      "id": "claude-baseline-2026-11-20T11-18-01-792Z",
      "provider": "claude",
      "promptId": "baseline",
      "prompt": "Manage the Back Kitchen across two shifts. Maximize delivered order value within all constraints. Ask teammates for missing facts, schedule orders, and finish each shift. Leave a brief operational handoff for your next session.",
      "runnerVersion": "2.1.325 (Claude Code)",
      "model": "claude-opus-5-5",
      "modelSource": "CLI modelUsage response",
      "status": "finished",
      "startedAt": "2026-11-20T11:18:01.922Z",
      "endedAt": "2026-11-20T11:19:22.598Z",
      "budgets": {
        "maxCalls": 8,
        "maxCallSeconds": 120,
        "maxActionsPerShift": 40
      },
      "usage": {
        "inputTokens": 8,
        "outputTokens": 4264,
        "cachedInputTokens": 5892,
        "costUsd": 0.2162744
      },
      "modelCalls": 4,
      "elapsedMs": 49020,
      "episodes": [
        {
          "index": 0,
          "seed": 1,
          "status": "finished",
          "scorePercent": 100,
          "earnedValue": 118,
          "optimalValue": 118,
          "deliveredOrders": 4,
          "invalidActions": 0,
          "actions": 7,
          "questions": 2,
          "callCount": 2,
          "checkpoints": 2,
          "events": [
            {
              "sequence": 1,
              "type": "ask",
              "ok": true,
              "teammate": "kitchen"
            },
            {
              "sequence": 2,
              "type": "ask",
              "ok": true,
              "teammate": "courier"
            },
            {
              "sequence": 3,
              "type": "assign",
              "ok": true,
              "orderId": "order-5",
              "slotId": "slot-2-2"
            },
            {
              "sequence": 4,
              "type": "assign",
              "ok": true,
              "orderId": "order-6",
              "slotId": "slot-3-1"
            },
            {
              "sequence": 5,
              "type": "assign",
              "ok": true,
              "orderId": "order-2",
              "slotId": "slot-2-1"
            },
            {
              "sequence": 6,
              "type": "assign",
              "ok": true,
              "orderId": "order-7",
              "slotId": "slot-3-2"
            },
            {
              "sequence": 7,
              "type": "finish",
              "ok": true
            }
          ],
          "updates": [
            {
              "call": 1,
              "summary": "Shift 1: I've asked both teammates for their briefings. Kitchen should cover oven/recipe fit, bake windows and ingredient budgets. Courier should cover which destinations each courier serves, travel times and credit costs. No orders are assigned yet. Next step: once the briefings come back, work out the best on-time schedule for all 8 orders across 8 slots (4 waves × wood/electric ovens, couriers alba/nico). High-value orders to prioritise are order-7 (35), order-5 (31) and order-6 (30). Order-1 (east, deadline 9) and order-8 (south, deadline 19) can only use wave 1 or 2 slots. After that, assign the orders and finish the shift."
            },
            {
              "call": 2,
              "summary": "Shift 1 assigned and finished. Every order uses 1 cheese and there are only 4 cheese, so we can deliver at most 4 orders. I picked the most valuable set that fits the 23-credit budget: order-7 (35), order-5 (31), order-6 (30) and order-2 (22). Total value is 118 for 22 credits. Plan: order-5 in slot-2-2 (electric/alba, north, arrives about 17, deadline 29). Order-6 in slot-3-1 (wood/alba, north, about 27, deadline 29). Order-2 in slot-2-1 (wood/nico, south, about 19, deadline 39; pineapple can only bake in the wood oven). Order-7 in slot-3-2 (electric/nico, south, about 29, deadline 39). Arrival times assume arrival = dispatch time + one-way travel time. Left out: order-1, order-3, order-4 and order-8 (not enough cheese). Next shift: re-check the briefings, since stock, budget, routes or oven fit may change. Then use the same method: cheese and dough set the order count, choose the highest value that fits the credit budget, and match each order to a slot by courier destination, oven recipe and deadline."
            }
          ],
          "callRecords": [
            {
              "call": 1,
              "status": "completed",
              "elapsedMs": 6309
            },
            {
              "call": 2,
              "status": "completed",
              "elapsedMs": 15927
            }
          ],
          "scenario": {
            "budgetCredits": 23,
            "stock": {
              "dough": 5,
              "cheese": 4,
              "mushroom": 4,
              "pineapple": 2
            },
            "recipes": [
              {
                "id": "margherita",
                "bakeMinutes": 3,
                "costCredits": 2,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 0
                }
              },
              {
                "id": "mushroom",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 1,
                  "pineapple": 0
                }
              },
              {
                "id": "pineapple",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 1
                }
              },
              {
                "id": "garden",
                "bakeMinutes": 5,
                "costCredits": 4,
                "ingredients": {
                  "dough": 1,
                  "cheese": 0,
                  "mushroom": 2,
                  "pineapple": 0
                }
              }
            ],
            "orders": [
              {
                "id": "order-1",
                "recipeId": "margherita",
                "destination": "east",
                "readyAt": 0,
                "deadline": 9,
                "value": 16
              },
              {
                "id": "order-2",
                "recipeId": "pineapple",
                "destination": "south",
                "readyAt": 10,
                "deadline": 39,
                "value": 22
              },
              {
                "id": "order-3",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 19,
                "value": 15
              },
              {
                "id": "order-4",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 39,
                "value": 18
              },
              {
                "id": "order-5",
                "recipeId": "margherita",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 31
              },
              {
                "id": "order-6",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 30
              },
              {
                "id": "order-7",
                "recipeId": "margherita",
                "destination": "south",
                "readyAt": 10,
                "deadline": 39,
                "value": 35
              },
              {
                "id": "order-8",
                "recipeId": "margherita",
                "destination": "south",
                "readyAt": 0,
                "deadline": 19,
                "value": 21
              }
            ],
            "slots": [
              {
                "id": "slot-1-1",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-1-2",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-2-1",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-2-2",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "electric",
                "courierId": "alba"
              },
              {
                "id": "slot-3-1",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-3-2",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-4-1",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-4-2",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "electric",
                "courierId": "alba"
              }
            ],
            "ovens": [
              {
                "id": "wood",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "pineapple"
                ]
              },
              {
                "id": "electric",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "garden"
                ]
              }
            ],
            "couriers": [
              {
                "id": "alba",
                "destinations": [
                  "north",
                  "east"
                ]
              },
              {
                "id": "nico",
                "destinations": [
                  "east",
                  "south"
                ]
              }
            ],
            "routes": [
              {
                "destination": "north",
                "oneWayMinutes": 2,
                "feeCredits": 2
              },
              {
                "destination": "east",
                "oneWayMinutes": 3,
                "feeCredits": 3
              },
              {
                "destination": "south",
                "oneWayMinutes": 4,
                "feeCredits": 4
              }
            ]
          },
          "referencePlan": [
            {
              "orderId": "order-5",
              "slotId": "slot-2-2"
            },
            {
              "orderId": "order-6",
              "slotId": "slot-3-1"
            },
            {
              "orderId": "order-7",
              "slotId": "slot-3-2"
            },
            {
              "orderId": "order-2",
              "slotId": "slot-4-1"
            }
          ]
        },
        {
          "index": 1,
          "seed": 17,
          "status": "finished",
          "scorePercent": 100,
          "earnedValue": 114,
          "optimalValue": 114,
          "deliveredOrders": 4,
          "invalidActions": 0,
          "actions": 7,
          "questions": 2,
          "callCount": 2,
          "checkpoints": 2,
          "events": [
            {
              "sequence": 1,
              "type": "ask",
              "ok": true,
              "teammate": "kitchen"
            },
            {
              "sequence": 2,
              "type": "ask",
              "ok": true,
              "teammate": "courier"
            },
            {
              "sequence": 3,
              "type": "assign",
              "ok": true,
              "orderId": "order-1",
              "slotId": "slot-1-2"
            },
            {
              "sequence": 4,
              "type": "assign",
              "ok": true,
              "orderId": "order-3",
              "slotId": "slot-1-1"
            },
            {
              "sequence": 5,
              "type": "assign",
              "ok": true,
              "orderId": "order-2",
              "slotId": "slot-2-2"
            },
            {
              "sequence": 6,
              "type": "assign",
              "ok": true,
              "orderId": "order-7",
              "slotId": "slot-3-1"
            },
            {
              "sequence": 7,
              "type": "finish",
              "ok": true
            }
          ],
          "updates": [
            {
              "call": 3,
              "summary": "Shift 2: I've asked the kitchen and the courier for their briefings. No orders are assigned yet. The orders and destinations are different from shift 1, so I'll re-check stock, budget, oven fit and routes instead of reusing the shift-1 plan. Orders: order-1 (margherita, east, deadline 9, value 20), order-2 (mushroom, east, deadline 29, 31), order-3 (mushroom, north, deadline 29, 29), order-4 (mushroom, east, ready 20, deadline 39, 25), order-5 (pineapple, east, ready 10, deadline 19, 13), order-6 (garden, north, ready 10, deadline 29, 26), order-7 (margherita, north, ready 20, deadline 29, 34), order-8 (mushroom, north, ready 20, deadline 29, 21). Order-7 is ready at 20 and due at 29, so it needs a wave-3 slot. Order-1 can only make its deadline in wave 1. Order-5 needs wave 2. Next step: once the briefings come back, find the scarce ingredient and the credit limit, choose the highest-value set of orders that fits, match each one to a slot by oven, courier route and deadline, then assign the orders and finish."
            },
            {
              "call": 4,
              "summary": "Shift 2 is assigned and finished. Cheese (4) limits us to 4 orders and mushroom (2) limits the mix. One option was garden order-6, which uses 2 mushroom, plus orders 1, 5 and 7, for a value of 93. The better option was two mushroom orders, which gives 114. Chosen set: order-7 (34), order-2 (31), order-3 (29) and order-1 (20). Total value is 114 for 20 of 24 credits. Plan: order-1 in slot-1-2 (electric/nico, east, arrives at 8, deadline 9). Order-3 in slot-1-1 (wood/alba, north, arrives at 7, deadline 29). Order-2 in slot-2-2 (electric/alba, east, arrives at 18, deadline 29). Order-7 in slot-3-1 (wood/alba, north, arrives at 27, deadline 29). Left out: order-4, order-5, order-6 and order-8, because there isn't enough cheese or mushroom. Handoff for next session: get both briefings first. Then find the ingredient that limits the order count (cheese, dough or a topping). List all feasible combinations, pick the highest value within the credit budget, and match each order to a slot by oven, courier destination, readyAt/startAt and arrival = dispatch time + one-way travel time against the deadline."
            }
          ],
          "callRecords": [
            {
              "call": 3,
              "status": "completed",
              "elapsedMs": 7575
            },
            {
              "call": 4,
              "status": "completed",
              "elapsedMs": 19209
            }
          ],
          "scenario": {
            "budgetCredits": 24,
            "stock": {
              "dough": 5,
              "cheese": 4,
              "mushroom": 2,
              "pineapple": 1
            },
            "recipes": [
              {
                "id": "margherita",
                "bakeMinutes": 3,
                "costCredits": 2,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 0
                }
              },
              {
                "id": "mushroom",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 1,
                  "pineapple": 0
                }
              },
              {
                "id": "pineapple",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 1
                }
              },
              {
                "id": "garden",
                "bakeMinutes": 5,
                "costCredits": 4,
                "ingredients": {
                  "dough": 1,
                  "cheese": 0,
                  "mushroom": 2,
                  "pineapple": 0
                }
              }
            ],
            "orders": [
              {
                "id": "order-1",
                "recipeId": "margherita",
                "destination": "east",
                "readyAt": 0,
                "deadline": 9,
                "value": 20
              },
              {
                "id": "order-2",
                "recipeId": "mushroom",
                "destination": "east",
                "readyAt": 0,
                "deadline": 29,
                "value": 31
              },
              {
                "id": "order-3",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 0,
                "deadline": 29,
                "value": 29
              },
              {
                "id": "order-4",
                "recipeId": "mushroom",
                "destination": "east",
                "readyAt": 20,
                "deadline": 39,
                "value": 25
              },
              {
                "id": "order-5",
                "recipeId": "pineapple",
                "destination": "east",
                "readyAt": 10,
                "deadline": 19,
                "value": 13
              },
              {
                "id": "order-6",
                "recipeId": "garden",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 26
              },
              {
                "id": "order-7",
                "recipeId": "margherita",
                "destination": "north",
                "readyAt": 20,
                "deadline": 29,
                "value": 34
              },
              {
                "id": "order-8",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 20,
                "deadline": 29,
                "value": 21
              }
            ],
            "slots": [
              {
                "id": "slot-1-1",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-1-2",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-2-1",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-2-2",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "electric",
                "courierId": "alba"
              },
              {
                "id": "slot-3-1",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-3-2",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-4-1",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-4-2",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "electric",
                "courierId": "alba"
              }
            ],
            "ovens": [
              {
                "id": "wood",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "pineapple"
                ]
              },
              {
                "id": "electric",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "garden"
                ]
              }
            ],
            "couriers": [
              {
                "id": "alba",
                "destinations": [
                  "north",
                  "east"
                ]
              },
              {
                "id": "nico",
                "destinations": [
                  "east",
                  "south"
                ]
              }
            ],
            "routes": [
              {
                "destination": "north",
                "oneWayMinutes": 2,
                "feeCredits": 2
              },
              {
                "destination": "east",
                "oneWayMinutes": 3,
                "feeCredits": 3
              },
              {
                "destination": "south",
                "oneWayMinutes": 4,
                "feeCredits": 4
              }
            ]
          },
          "referencePlan": [
            {
              "orderId": "order-1",
              "slotId": "slot-1-2"
            },
            {
              "orderId": "order-3",
              "slotId": "slot-2-2"
            },
            {
              "orderId": "order-7",
              "slotId": "slot-3-1"
            },
            {
              "orderId": "order-2",
              "slotId": "slot-3-2"
            }
          ]
        }
      ],
      "limitations": [
        "Two short static shifts with a process restart; not evidence of unattended multi-hour management.",
        "Local CLI JSON-action adapter; no native MCP integration or production registrations.",
        "Scripted teammates, fixed scenario seeds, one campaign per prompt/client; not a general model ranking.",
        "Different client system prompts and default models; Codex may expose unused write tools under read-only permissions. Tool attempts invalidate the campaign.",
        "Token accounting differs between providers; Claude cost is an API-equivalent estimate, not an actual subscription charge.",
        "Public summaries are generated before their action batch executes. The next shift receives recent summaries, not the previous final report.",
        "Usage counters retain provider-native meanings and may omit failed calls. Claude input excludes cache reads and writes; cache-write counts were not captured."
      ]
    },
    {
      "id": "claude-checklist-2026-11-20T11-20-19-221Z",
      "provider": "claude",
      "promptId": "checklist",
      "prompt": "Manage the Back Kitchen across two shifts. Before scheduling, obtain both teammate briefings. Check recipe, oven, courier zone, preparation time, deadline, ingredients and shared credits together. Compare feasible schedules for delivered value. Reconcile the returned state after each batch, avoid repeating committed actions, finish when satisfied, and leave a brief operational handoff for your next session.",
      "runnerVersion": "2.1.325 (Claude Code)",
      "model": "claude-opus-5-5",
      "modelSource": "CLI modelUsage response",
      "status": "finished",
      "startedAt": "2026-11-20T11:20:19.472Z",
      "endedAt": "2026-11-20T11:21:21.430Z",
      "budgets": {
        "maxCalls": 8,
        "maxCallSeconds": 120,
        "maxActionsPerShift": 40
      },
      "usage": {
        "inputTokens": 8,
        "outputTokens": 4237,
        "cachedInputTokens": 9272,
        "costUsd": 0.1878904
      },
      "modelCalls": 4,
      "elapsedMs": 50168,
      "episodes": [
        {
          "index": 0,
          "seed": 1,
          "status": "finished",
          "scorePercent": 100,
          "earnedValue": 118,
          "optimalValue": 118,
          "deliveredOrders": 4,
          "invalidActions": 0,
          "actions": 7,
          "questions": 2,
          "callCount": 2,
          "checkpoints": 2,
          "events": [
            {
              "sequence": 1,
              "type": "ask",
              "ok": true,
              "teammate": "kitchen"
            },
            {
              "sequence": 2,
              "type": "ask",
              "ok": true,
              "teammate": "courier"
            },
            {
              "sequence": 3,
              "type": "assign",
              "ok": true,
              "orderId": "order-5",
              "slotId": "slot-2-2"
            },
            {
              "sequence": 4,
              "type": "assign",
              "ok": true,
              "orderId": "order-6",
              "slotId": "slot-3-1"
            },
            {
              "sequence": 5,
              "type": "assign",
              "ok": true,
              "orderId": "order-2",
              "slotId": "slot-2-1"
            },
            {
              "sequence": 6,
              "type": "assign",
              "ok": true,
              "orderId": "order-7",
              "slotId": "slot-3-2"
            },
            {
              "sequence": 7,
              "type": "finish",
              "ok": true
            }
          ],
          "updates": [
            {
              "call": 1,
              "summary": "Shift 1: I asked the kitchen and courier teammates for their briefings. No orders are assigned and no credits or ingredients are used yet. Next step: read both briefings. Then check each order's recipe and oven fit, bake window, courier zone, ready time, deadline, ingredients and shared credits. After that, pick the feasible schedule with the most delivered value, assign it, and finish."
            },
            {
              "call": 2,
              "summary": "Shift 1: I read both briefings. Cheese (4) caps the shift at 4 orders, and credits (23) limit which 4 we can take. Best feasible plan, worth 118: order-5 margherita→north in slot-2-2 (electric/alba, arrives 17, deadline 29); order-6 mushroom→north in slot-3-1 (wood/alba, arrives 27, deadline 29); order-2 pineapple→south in slot-2-1 (wood/nico, arrives 19, deadline 39); order-7 margherita→south in slot-3-2 (electric/nico, arrives 29, deadline 39). These use 22 of 23 credits, all 4 cheese, 4 dough, 1 mushroom and 1 pineapple. The next-best option, order-8 in place of order-2, is worth 117. All other orders are dropped because of cheese and credit limits. This batch assigns those 4 orders and then finishes. Next session: check the shift-1 report and compare it with the reference plan. Then ask both teammates for their shift-2 briefings and apply the same checks: recipe/oven fit, bake window, courier zone, ready time, deadline, ingredients and credits."
            }
          ],
          "callRecords": [
            {
              "call": 1,
              "status": "completed",
              "elapsedMs": 6594
            },
            {
              "call": 2,
              "status": "completed",
              "elapsedMs": 19797
            }
          ],
          "scenario": {
            "budgetCredits": 23,
            "stock": {
              "dough": 5,
              "cheese": 4,
              "mushroom": 4,
              "pineapple": 2
            },
            "recipes": [
              {
                "id": "margherita",
                "bakeMinutes": 3,
                "costCredits": 2,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 0
                }
              },
              {
                "id": "mushroom",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 1,
                  "pineapple": 0
                }
              },
              {
                "id": "pineapple",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 1
                }
              },
              {
                "id": "garden",
                "bakeMinutes": 5,
                "costCredits": 4,
                "ingredients": {
                  "dough": 1,
                  "cheese": 0,
                  "mushroom": 2,
                  "pineapple": 0
                }
              }
            ],
            "orders": [
              {
                "id": "order-1",
                "recipeId": "margherita",
                "destination": "east",
                "readyAt": 0,
                "deadline": 9,
                "value": 16
              },
              {
                "id": "order-2",
                "recipeId": "pineapple",
                "destination": "south",
                "readyAt": 10,
                "deadline": 39,
                "value": 22
              },
              {
                "id": "order-3",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 19,
                "value": 15
              },
              {
                "id": "order-4",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 39,
                "value": 18
              },
              {
                "id": "order-5",
                "recipeId": "margherita",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 31
              },
              {
                "id": "order-6",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 30
              },
              {
                "id": "order-7",
                "recipeId": "margherita",
                "destination": "south",
                "readyAt": 10,
                "deadline": 39,
                "value": 35
              },
              {
                "id": "order-8",
                "recipeId": "margherita",
                "destination": "south",
                "readyAt": 0,
                "deadline": 19,
                "value": 21
              }
            ],
            "slots": [
              {
                "id": "slot-1-1",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-1-2",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-2-1",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-2-2",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "electric",
                "courierId": "alba"
              },
              {
                "id": "slot-3-1",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-3-2",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-4-1",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-4-2",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "electric",
                "courierId": "alba"
              }
            ],
            "ovens": [
              {
                "id": "wood",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "pineapple"
                ]
              },
              {
                "id": "electric",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "garden"
                ]
              }
            ],
            "couriers": [
              {
                "id": "alba",
                "destinations": [
                  "north",
                  "east"
                ]
              },
              {
                "id": "nico",
                "destinations": [
                  "east",
                  "south"
                ]
              }
            ],
            "routes": [
              {
                "destination": "north",
                "oneWayMinutes": 2,
                "feeCredits": 2
              },
              {
                "destination": "east",
                "oneWayMinutes": 3,
                "feeCredits": 3
              },
              {
                "destination": "south",
                "oneWayMinutes": 4,
                "feeCredits": 4
              }
            ]
          },
          "referencePlan": [
            {
              "orderId": "order-5",
              "slotId": "slot-2-2"
            },
            {
              "orderId": "order-6",
              "slotId": "slot-3-1"
            },
            {
              "orderId": "order-7",
              "slotId": "slot-3-2"
            },
            {
              "orderId": "order-2",
              "slotId": "slot-4-1"
            }
          ]
        },
        {
          "index": 1,
          "seed": 17,
          "status": "finished",
          "scorePercent": 100,
          "earnedValue": 114,
          "optimalValue": 114,
          "deliveredOrders": 4,
          "invalidActions": 0,
          "actions": 7,
          "questions": 2,
          "callCount": 2,
          "checkpoints": 2,
          "events": [
            {
              "sequence": 1,
              "type": "ask",
              "ok": true,
              "teammate": "kitchen"
            },
            {
              "sequence": 2,
              "type": "ask",
              "ok": true,
              "teammate": "courier"
            },
            {
              "sequence": 3,
              "type": "assign",
              "ok": true,
              "orderId": "order-7",
              "slotId": "slot-3-1"
            },
            {
              "sequence": 4,
              "type": "assign",
              "ok": true,
              "orderId": "order-3",
              "slotId": "slot-1-1"
            },
            {
              "sequence": 5,
              "type": "assign",
              "ok": true,
              "orderId": "order-2",
              "slotId": "slot-2-1"
            },
            {
              "sequence": 6,
              "type": "assign",
              "ok": true,
              "orderId": "order-1",
              "slotId": "slot-1-2"
            },
            {
              "sequence": 7,
              "type": "finish",
              "ok": true
            }
          ],
          "updates": [
            {
              "call": 3,
              "summary": "Shift 2: Shift 1 was finalized with the 4-order plan worth 118, but its report isn't shown in this observation, so I can't yet compare it with the reference plan. Shift 2 has 8 orders and 8 slots. I asked the kitchen and courier teammates for their briefings. Nothing is assigned yet, and no credits or ingredients are used. Next step: read both briefings. Then check each order's recipe and oven fit, bake window, courier zone, ready time, deadline, ingredients and shared credits. Note: order-1 (east, deadline 9) can only go in a wave-1 slot, and order-7 and order-8 (ready at 20, deadline 29) can only go in wave-3 slots. After that, compare feasible schedules for delivered value, assign the best one, and finish."
            },
            {
              "call": 4,
              "summary": "Shift 2: I read both briefings. Cheese (4) limits us to 4 cheese orders, and we only have 2 mushrooms. The garden pizza, order-6, needs both mushrooms. The best plan with it (orders 6, 1, 5 and 7) is worth 93. The best plan without it is worth 114, so I chose that one: order-1 margherita→east in slot-1-2 (electric/nico, arrives 8, deadline 9); order-3 mushroom→north in slot-1-1 (wood/alba, arrives 7, deadline 29); order-2 mushroom→east in slot-2-1 (wood/nico, arrives 18, deadline 29); order-7 margherita→north in slot-3-1 (wood/alba, arrives 27, deadline 29). This uses 20 of 24 credits, all 4 cheese, 4 of 5 dough, both mushrooms and no pineapple. Orders 4 and 8 are dropped because there are no mushrooms left. Order-5 is dropped because there is no cheese left, and order-6 because there are no mushrooms left. This batch assigns those 4 orders and then finishes. Handoff: after finishing, check the shift-2 report against the reference plan (expected 114), and do the same for shift 1 (expected 118). If any assignment was rejected, re-check its oven, courier zone and timing."
            }
          ],
          "callRecords": [
            {
              "call": 3,
              "status": "completed",
              "elapsedMs": 5727
            },
            {
              "call": 4,
              "status": "completed",
              "elapsedMs": 18050
            }
          ],
          "scenario": {
            "budgetCredits": 24,
            "stock": {
              "dough": 5,
              "cheese": 4,
              "mushroom": 2,
              "pineapple": 1
            },
            "recipes": [
              {
                "id": "margherita",
                "bakeMinutes": 3,
                "costCredits": 2,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 0
                }
              },
              {
                "id": "mushroom",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 1,
                  "pineapple": 0
                }
              },
              {
                "id": "pineapple",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 1
                }
              },
              {
                "id": "garden",
                "bakeMinutes": 5,
                "costCredits": 4,
                "ingredients": {
                  "dough": 1,
                  "cheese": 0,
                  "mushroom": 2,
                  "pineapple": 0
                }
              }
            ],
            "orders": [
              {
                "id": "order-1",
                "recipeId": "margherita",
                "destination": "east",
                "readyAt": 0,
                "deadline": 9,
                "value": 20
              },
              {
                "id": "order-2",
                "recipeId": "mushroom",
                "destination": "east",
                "readyAt": 0,
                "deadline": 29,
                "value": 31
              },
              {
                "id": "order-3",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 0,
                "deadline": 29,
                "value": 29
              },
              {
                "id": "order-4",
                "recipeId": "mushroom",
                "destination": "east",
                "readyAt": 20,
                "deadline": 39,
                "value": 25
              },
              {
                "id": "order-5",
                "recipeId": "pineapple",
                "destination": "east",
                "readyAt": 10,
                "deadline": 19,
                "value": 13
              },
              {
                "id": "order-6",
                "recipeId": "garden",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 26
              },
              {
                "id": "order-7",
                "recipeId": "margherita",
                "destination": "north",
                "readyAt": 20,
                "deadline": 29,
                "value": 34
              },
              {
                "id": "order-8",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 20,
                "deadline": 29,
                "value": 21
              }
            ],
            "slots": [
              {
                "id": "slot-1-1",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-1-2",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-2-1",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-2-2",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "electric",
                "courierId": "alba"
              },
              {
                "id": "slot-3-1",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-3-2",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-4-1",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-4-2",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "electric",
                "courierId": "alba"
              }
            ],
            "ovens": [
              {
                "id": "wood",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "pineapple"
                ]
              },
              {
                "id": "electric",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "garden"
                ]
              }
            ],
            "couriers": [
              {
                "id": "alba",
                "destinations": [
                  "north",
                  "east"
                ]
              },
              {
                "id": "nico",
                "destinations": [
                  "east",
                  "south"
                ]
              }
            ],
            "routes": [
              {
                "destination": "north",
                "oneWayMinutes": 2,
                "feeCredits": 2
              },
              {
                "destination": "east",
                "oneWayMinutes": 3,
                "feeCredits": 3
              },
              {
                "destination": "south",
                "oneWayMinutes": 4,
                "feeCredits": 4
              }
            ]
          },
          "referencePlan": [
            {
              "orderId": "order-1",
              "slotId": "slot-1-2"
            },
            {
              "orderId": "order-3",
              "slotId": "slot-2-2"
            },
            {
              "orderId": "order-7",
              "slotId": "slot-3-1"
            },
            {
              "orderId": "order-2",
              "slotId": "slot-3-2"
            }
          ]
        }
      ],
      "limitations": [
        "Two short static shifts with a process restart; not evidence of unattended multi-hour management.",
        "Local CLI JSON-action adapter; no native MCP integration or production registrations.",
        "Scripted teammates, fixed scenario seeds, one campaign per prompt/client; not a general model ranking.",
        "Different client system prompts and default models; Codex may expose unused write tools under read-only permissions. Tool attempts invalidate the campaign.",
        "Token accounting differs between providers; Claude cost is an API-equivalent estimate, not an actual subscription charge.",
        "Public summaries are generated before their action batch executes. The next shift receives recent summaries, not the previous final report.",
        "Usage counters retain provider-native meanings and may omit failed calls. Claude input excludes cache reads and writes; cache-write counts were not captured."
      ]
    },
    {
      "id": "codex-baseline-2026-11-20T11-18-01-794Z",
      "provider": "codex",
      "promptId": "baseline",
      "prompt": "Manage the Back Kitchen across two shifts. Maximize delivered order value within all constraints. Ask teammates for missing facts, schedule orders, and finish each shift. Leave a brief operational handoff for your next session.",
      "runnerVersion": "codex-cli 0.170.0",
      "model": null,
      "modelSource": "not reported by CLI",
      "status": "failed",
      "startedAt": "2026-11-20T11:18:02.081Z",
      "endedAt": "2026-11-20T11:18:11.059Z",
      "budgets": {
        "maxCalls": 8,
        "maxCallSeconds": 120,
        "maxActionsPerShift": 40
      },
      "usage": {
        "inputTokens": null,
        "outputTokens": null,
        "cachedInputTokens": null,
        "costUsd": null
      },
      "modelCalls": 1,
      "elapsedMs": 8965,
      "episodes": [
        {
          "index": 0,
          "seed": 1,
          "status": "failed",
          "scorePercent": null,
          "earnedValue": null,
          "optimalValue": null,
          "deliveredOrders": null,
          "invalidActions": 0,
          "actions": 0,
          "questions": 0,
          "callCount": 1,
          "checkpoints": 0,
          "events": [],
          "updates": [],
          "callRecords": [
            {
              "call": 1,
              "status": "failed",
              "error": "tool_attempt_rejected",
              "elapsedMs": 8965
            }
          ],
          "scenario": {
            "budgetCredits": 23,
            "stock": {
              "dough": 5,
              "cheese": 4,
              "mushroom": 4,
              "pineapple": 2
            },
            "recipes": [
              {
                "id": "margherita",
                "bakeMinutes": 3,
                "costCredits": 2,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 0
                }
              },
              {
                "id": "mushroom",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 1,
                  "pineapple": 0
                }
              },
              {
                "id": "pineapple",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 1
                }
              },
              {
                "id": "garden",
                "bakeMinutes": 5,
                "costCredits": 4,
                "ingredients": {
                  "dough": 1,
                  "cheese": 0,
                  "mushroom": 2,
                  "pineapple": 0
                }
              }
            ],
            "orders": [
              {
                "id": "order-1",
                "recipeId": "margherita",
                "destination": "east",
                "readyAt": 0,
                "deadline": 9,
                "value": 16
              },
              {
                "id": "order-2",
                "recipeId": "pineapple",
                "destination": "south",
                "readyAt": 10,
                "deadline": 39,
                "value": 22
              },
              {
                "id": "order-3",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 19,
                "value": 15
              },
              {
                "id": "order-4",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 39,
                "value": 18
              },
              {
                "id": "order-5",
                "recipeId": "margherita",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 31
              },
              {
                "id": "order-6",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 30
              },
              {
                "id": "order-7",
                "recipeId": "margherita",
                "destination": "south",
                "readyAt": 10,
                "deadline": 39,
                "value": 35
              },
              {
                "id": "order-8",
                "recipeId": "margherita",
                "destination": "south",
                "readyAt": 0,
                "deadline": 19,
                "value": 21
              }
            ],
            "slots": [
              {
                "id": "slot-1-1",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-1-2",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-2-1",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-2-2",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "electric",
                "courierId": "alba"
              },
              {
                "id": "slot-3-1",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-3-2",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-4-1",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-4-2",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "electric",
                "courierId": "alba"
              }
            ],
            "ovens": [
              {
                "id": "wood",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "pineapple"
                ]
              },
              {
                "id": "electric",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "garden"
                ]
              }
            ],
            "couriers": [
              {
                "id": "alba",
                "destinations": [
                  "north",
                  "east"
                ]
              },
              {
                "id": "nico",
                "destinations": [
                  "east",
                  "south"
                ]
              }
            ],
            "routes": [
              {
                "destination": "north",
                "oneWayMinutes": 2,
                "feeCredits": 2
              },
              {
                "destination": "east",
                "oneWayMinutes": 3,
                "feeCredits": 3
              },
              {
                "destination": "south",
                "oneWayMinutes": 4,
                "feeCredits": 4
              }
            ]
          },
          "referencePlan": []
        },
        {
          "index": 1,
          "seed": 17,
          "status": "pending",
          "scorePercent": null,
          "earnedValue": null,
          "optimalValue": null,
          "deliveredOrders": null,
          "invalidActions": 0,
          "actions": 0,
          "questions": 0,
          "callCount": 0,
          "checkpoints": 0,
          "events": [],
          "updates": [],
          "callRecords": [],
          "scenario": {
            "budgetCredits": 24,
            "stock": {
              "dough": 5,
              "cheese": 4,
              "mushroom": 2,
              "pineapple": 1
            },
            "recipes": [
              {
                "id": "margherita",
                "bakeMinutes": 3,
                "costCredits": 2,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 0
                }
              },
              {
                "id": "mushroom",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 1,
                  "pineapple": 0
                }
              },
              {
                "id": "pineapple",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 1
                }
              },
              {
                "id": "garden",
                "bakeMinutes": 5,
                "costCredits": 4,
                "ingredients": {
                  "dough": 1,
                  "cheese": 0,
                  "mushroom": 2,
                  "pineapple": 0
                }
              }
            ],
            "orders": [
              {
                "id": "order-1",
                "recipeId": "margherita",
                "destination": "east",
                "readyAt": 0,
                "deadline": 9,
                "value": 20
              },
              {
                "id": "order-2",
                "recipeId": "mushroom",
                "destination": "east",
                "readyAt": 0,
                "deadline": 29,
                "value": 31
              },
              {
                "id": "order-3",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 0,
                "deadline": 29,
                "value": 29
              },
              {
                "id": "order-4",
                "recipeId": "mushroom",
                "destination": "east",
                "readyAt": 20,
                "deadline": 39,
                "value": 25
              },
              {
                "id": "order-5",
                "recipeId": "pineapple",
                "destination": "east",
                "readyAt": 10,
                "deadline": 19,
                "value": 13
              },
              {
                "id": "order-6",
                "recipeId": "garden",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 26
              },
              {
                "id": "order-7",
                "recipeId": "margherita",
                "destination": "north",
                "readyAt": 20,
                "deadline": 29,
                "value": 34
              },
              {
                "id": "order-8",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 20,
                "deadline": 29,
                "value": 21
              }
            ],
            "slots": [
              {
                "id": "slot-1-1",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-1-2",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-2-1",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-2-2",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "electric",
                "courierId": "alba"
              },
              {
                "id": "slot-3-1",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-3-2",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-4-1",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-4-2",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "electric",
                "courierId": "alba"
              }
            ],
            "ovens": [
              {
                "id": "wood",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "pineapple"
                ]
              },
              {
                "id": "electric",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "garden"
                ]
              }
            ],
            "couriers": [
              {
                "id": "alba",
                "destinations": [
                  "north",
                  "east"
                ]
              },
              {
                "id": "nico",
                "destinations": [
                  "east",
                  "south"
                ]
              }
            ],
            "routes": [
              {
                "destination": "north",
                "oneWayMinutes": 2,
                "feeCredits": 2
              },
              {
                "destination": "east",
                "oneWayMinutes": 3,
                "feeCredits": 3
              },
              {
                "destination": "south",
                "oneWayMinutes": 4,
                "feeCredits": 4
              }
            ]
          },
          "referencePlan": []
        }
      ],
      "limitations": [
        "Two short static shifts with a process restart; not evidence of unattended multi-hour management.",
        "Local CLI JSON-action adapter; no native MCP integration or production registrations.",
        "Scripted teammates, fixed scenario seeds, one campaign per prompt/client; not a general model ranking.",
        "Different client system prompts and default models; Codex may expose unused write tools under read-only permissions. Tool attempts invalidate the campaign.",
        "Token accounting differs between providers; Claude cost is an API-equivalent estimate, not an actual subscription charge.",
        "Campaign stopped: tool_attempt_rejected. No automatic retry.",
        "Setup failure: the initial harness classified unfamiliar CLI notifications as tool attempts. The checklist failure identified a non-fatal error notification. These records do not establish that Codex used a tool; corrected-adapter trials are retained separately.",
        "Public summaries are generated before their action batch executes. The next shift receives recent summaries, not the previous final report.",
        "Usage counters retain provider-native meanings and may omit failed calls. Claude input excludes cache reads and writes; cache-write counts were not captured."
      ]
    },
    {
      "id": "codex-checklist-2026-11-20T11-18-55-798Z",
      "provider": "codex",
      "promptId": "checklist",
      "prompt": "Manage the Back Kitchen across two shifts. Before scheduling, obtain both teammate briefings. Check recipe, oven, courier zone, preparation time, deadline, ingredients and shared credits together. Compare feasible schedules for delivered value. Reconcile the returned state after each batch, avoid repeating committed actions, finish when satisfied, and leave a brief operational handoff for your next session.",
      "runnerVersion": "codex-cli 0.170.0",
      "model": null,
      "modelSource": "not reported by CLI",
      "status": "failed",
      "startedAt": "2026-11-20T11:18:56.200Z",
      "endedAt": "2026-11-20T11:18:59.818Z",
      "budgets": {
        "maxCalls": 8,
        "maxCallSeconds": 120,
        "maxActionsPerShift": 40
      },
      "usage": {
        "inputTokens": null,
        "outputTokens": null,
        "cachedInputTokens": null,
        "costUsd": null
      },
      "modelCalls": 1,
      "elapsedMs": 3607,
      "episodes": [
        {
          "index": 0,
          "seed": 1,
          "status": "failed",
          "scorePercent": null,
          "earnedValue": null,
          "optimalValue": null,
          "deliveredOrders": null,
          "invalidActions": 0,
          "actions": 0,
          "questions": 0,
          "callCount": 1,
          "checkpoints": 0,
          "events": [],
          "updates": [],
          "callRecords": [
            {
              "call": 1,
              "status": "failed",
              "error": "tool_attempt_rejected_error",
              "elapsedMs": 3607
            }
          ],
          "scenario": {
            "budgetCredits": 23,
            "stock": {
              "dough": 5,
              "cheese": 4,
              "mushroom": 4,
              "pineapple": 2
            },
            "recipes": [
              {
                "id": "margherita",
                "bakeMinutes": 3,
                "costCredits": 2,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 0
                }
              },
              {
                "id": "mushroom",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 1,
                  "pineapple": 0
                }
              },
              {
                "id": "pineapple",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 1
                }
              },
              {
                "id": "garden",
                "bakeMinutes": 5,
                "costCredits": 4,
                "ingredients": {
                  "dough": 1,
                  "cheese": 0,
                  "mushroom": 2,
                  "pineapple": 0
                }
              }
            ],
            "orders": [
              {
                "id": "order-1",
                "recipeId": "margherita",
                "destination": "east",
                "readyAt": 0,
                "deadline": 9,
                "value": 16
              },
              {
                "id": "order-2",
                "recipeId": "pineapple",
                "destination": "south",
                "readyAt": 10,
                "deadline": 39,
                "value": 22
              },
              {
                "id": "order-3",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 19,
                "value": 15
              },
              {
                "id": "order-4",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 39,
                "value": 18
              },
              {
                "id": "order-5",
                "recipeId": "margherita",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 31
              },
              {
                "id": "order-6",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 30
              },
              {
                "id": "order-7",
                "recipeId": "margherita",
                "destination": "south",
                "readyAt": 10,
                "deadline": 39,
                "value": 35
              },
              {
                "id": "order-8",
                "recipeId": "margherita",
                "destination": "south",
                "readyAt": 0,
                "deadline": 19,
                "value": 21
              }
            ],
            "slots": [
              {
                "id": "slot-1-1",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-1-2",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-2-1",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-2-2",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "electric",
                "courierId": "alba"
              },
              {
                "id": "slot-3-1",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-3-2",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-4-1",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-4-2",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "electric",
                "courierId": "alba"
              }
            ],
            "ovens": [
              {
                "id": "wood",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "pineapple"
                ]
              },
              {
                "id": "electric",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "garden"
                ]
              }
            ],
            "couriers": [
              {
                "id": "alba",
                "destinations": [
                  "north",
                  "east"
                ]
              },
              {
                "id": "nico",
                "destinations": [
                  "east",
                  "south"
                ]
              }
            ],
            "routes": [
              {
                "destination": "north",
                "oneWayMinutes": 2,
                "feeCredits": 2
              },
              {
                "destination": "east",
                "oneWayMinutes": 3,
                "feeCredits": 3
              },
              {
                "destination": "south",
                "oneWayMinutes": 4,
                "feeCredits": 4
              }
            ]
          },
          "referencePlan": []
        },
        {
          "index": 1,
          "seed": 17,
          "status": "pending",
          "scorePercent": null,
          "earnedValue": null,
          "optimalValue": null,
          "deliveredOrders": null,
          "invalidActions": 0,
          "actions": 0,
          "questions": 0,
          "callCount": 0,
          "checkpoints": 0,
          "events": [],
          "updates": [],
          "callRecords": [],
          "scenario": {
            "budgetCredits": 24,
            "stock": {
              "dough": 5,
              "cheese": 4,
              "mushroom": 2,
              "pineapple": 1
            },
            "recipes": [
              {
                "id": "margherita",
                "bakeMinutes": 3,
                "costCredits": 2,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 0
                }
              },
              {
                "id": "mushroom",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 1,
                  "pineapple": 0
                }
              },
              {
                "id": "pineapple",
                "bakeMinutes": 4,
                "costCredits": 3,
                "ingredients": {
                  "dough": 1,
                  "cheese": 1,
                  "mushroom": 0,
                  "pineapple": 1
                }
              },
              {
                "id": "garden",
                "bakeMinutes": 5,
                "costCredits": 4,
                "ingredients": {
                  "dough": 1,
                  "cheese": 0,
                  "mushroom": 2,
                  "pineapple": 0
                }
              }
            ],
            "orders": [
              {
                "id": "order-1",
                "recipeId": "margherita",
                "destination": "east",
                "readyAt": 0,
                "deadline": 9,
                "value": 20
              },
              {
                "id": "order-2",
                "recipeId": "mushroom",
                "destination": "east",
                "readyAt": 0,
                "deadline": 29,
                "value": 31
              },
              {
                "id": "order-3",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 0,
                "deadline": 29,
                "value": 29
              },
              {
                "id": "order-4",
                "recipeId": "mushroom",
                "destination": "east",
                "readyAt": 20,
                "deadline": 39,
                "value": 25
              },
              {
                "id": "order-5",
                "recipeId": "pineapple",
                "destination": "east",
                "readyAt": 10,
                "deadline": 19,
                "value": 13
              },
              {
                "id": "order-6",
                "recipeId": "garden",
                "destination": "north",
                "readyAt": 10,
                "deadline": 29,
                "value": 26
              },
              {
                "id": "order-7",
                "recipeId": "margherita",
                "destination": "north",
                "readyAt": 20,
                "deadline": 29,
                "value": 34
              },
              {
                "id": "order-8",
                "recipeId": "mushroom",
                "destination": "north",
                "readyAt": 20,
                "deadline": 29,
                "value": 21
              }
            ],
            "slots": [
              {
                "id": "slot-1-1",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-1-2",
                "wave": 1,
                "startAt": 0,
                "dispatchAt": 5,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-2-1",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-2-2",
                "wave": 2,
                "startAt": 10,
                "dispatchAt": 15,
                "ovenId": "electric",
                "courierId": "alba"
              },
              {
                "id": "slot-3-1",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "wood",
                "courierId": "alba"
              },
              {
                "id": "slot-3-2",
                "wave": 3,
                "startAt": 20,
                "dispatchAt": 25,
                "ovenId": "electric",
                "courierId": "nico"
              },
              {
                "id": "slot-4-1",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "wood",
                "courierId": "nico"
              },
              {
                "id": "slot-4-2",
                "wave": 4,
                "startAt": 30,
                "dispatchAt": 35,
                "ovenId": "electric",
                "courierId": "alba"
              }
            ],
            "ovens": [
              {
                "id": "wood",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "pineapple"
                ]
              },
              {
                "id": "electric",
                "supportedRecipes": [
                  "margherita",
                  "mushroom",
                  "garden"
                ]
              }
            ],
            "couriers": [
              {
                "id": "alba",
                "destinations": [
                  "north",
                  "east"
                ]
              },
              {
                "id": "nico",
                "destinations": [
                  "east",
                  "south"
                ]
              }
            ],
            "routes": [
              {
                "destination": "north",
                "oneWayMinutes": 2,
                "feeCredits": 2
              },
              {
                "destination": "east",
                "oneWayMinutes": 3,
                "feeCredits": 3
              },
              {
                "destination": "south",
                "oneWayMinutes": 4,
                "feeCredits": 4
              }
            ]
          },
          "referencePlan": []
        }
      ],
      "limitations": [
        "Two short static shifts with a process restart; not evidence of unattended multi-hour management.",
        "Local CLI JSON-action adapter; no native MCP integration or production registrations.",
        "Scripted teammates, fixed scenario seeds, one campaign per prompt/client; not a general model ranking.",
        "Different client system prompts and default models; Codex may expose unused write tools under read-only permissions. Tool attempts invalidate the campaign.",
        "Token accounting differs between providers; Claude cost is an API-equivalent estimate, not an actual subscription charge.",
        "Campaign stopped: tool_attempt_rejected_error. No automatic retry.",
        "Setup failure: the initial harness classified unfamiliar CLI notifications as tool attempts. The checklist failure identified a non-fatal error notification. These records do not establish that Codex used a tool; corrected-adapter trials are retained separately.",
        "Public summaries are generated before their action batch executes. The next shift receives recent summaries, not the previous final report.",
        "Usage counters retain provider-native meanings and may omit failed calls. Claude input excludes cache reads and writes; cache-write counts were not captured."
      ]
    }
  ],
  "publication": {
    "reviewedHarness": {
      "repository": "https://github.com/marcuspuchalla/online-pizza-agent",
      "commit": "0f2426b27c330916078226c9d238165b7e3560d9",
      "sha256": {
        "lab/protocol.mjs": "8c4ee60fdc116cc6f1715ca023ce452ce83e49b4de7bfa75cec38d9197b64966",
        "lab/providers.mjs": "dc81ee092fd208ef1d4a0e4806647b55ec91f005dc166cc5fa2c02d07646b517",
        "lab/run.mjs": "e5c5dda37e92086ed02e31dc180f9c7ba28792b675c859d2269196723b7c6f5f",
        "lab/checkpoint.mjs": "7d1445daf7a57ff764b50ccd5222c5570b6d50df5a264f1c2f51a9e405368790",
        "lab/engine/shift-engine.mjs": "f1e7a7f6ce2dc0e57c053c0187e16459e0295240b93237d25230e75e77b4a85d"
      }
    },
    "note": "Final reviewed source. Initial failed Codex attempts used the earlier notification classifier described in their limitations; this is not a claim of identical harness bytes across all attempts. Simulator whitespace was normalized after runs without semantic changes.",
    "interpretation": "Four completed campaigns reached the exact optimum on both seeds; the task is an onboarding demonstration, not a discriminating model ranking. Two setup failures are retained."
  }
}
