{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://latenteval.ai/schemas/eval-set/case-profile/v1.json",
  "title": "Eval case profile, v1",
  "description": "A profile of the eval-set v1 schema: the object a row may carry at metadata.case. It records what a case tests, what correct and unacceptable behavior look like, which oracle applies, who may grade it, how reversible a failure is and how far it reaches, and where the case came from. It adds no top-level key to eval-set v1 and a row carrying it validates against eval-set v1 unchanged. The version lives in the path. Inside a published version only additive, optional changes are allowed; a breaking change publishes v2 beside this file, never over it.",
  "x-publishedOn": "2026-09-12",
  "type": "object",
  "properties": {
    "caseId": {
      "type": "string",
      "pattern": "\\S",
      "description": "The row's own id, repeated here so the profile travels intact when a vendor export flattens the row. A reader MUST keep it equal to the row's id; the validator reports a mismatch."
    },
    "scenario": {
      "type": "string",
      "pattern": "\\S",
      "description": "What is happening, in one or two sentences, from outside the system."
    },
    "actor": {
      "type": "string",
      "description": "Who or what produced the input: a named user type, an upstream system, a retrieved document."
    },
    "oracleType": {
      "enum": [
        "reference-answer",
        "acceptable-set",
        "rubric",
        "expected-tool-calls",
        "expected-final-state",
        "expected-behavior-only"
      ],
      "description": "Which form of expected behavior this case carries. It gates which evaluators may grade the case."
    },
    "expectedBehavior": {
      "type": "string",
      "pattern": "\\S",
      "description": "What the system should do. Prose where oracleType is expected-behavior-only or rubric; a pointer to the row's own expected field otherwise."
    },
    "unacceptableBehavior": {
      "type": "array",
      "items": { "type": "string" },
      "description": "The must-not-do list. The only field that survives a system rewrite unchanged, which is why it is separate from expectedBehavior."
    },
    "evaluationCriteria": {
      "type": "array",
      "items": { "type": "string" },
      "description": "The criteria a grader applies, one per entry. For a rubric oracle this is the rubric."
    },
    "reversibility": {
      "enum": ["R0", "R1", "R2", "R3"],
      "description": "Reversibility of the worst outcome if this case fails in production, in the vocabulary of the action-risk matrix. R0 changes no state; R3 is a point of no return."
    },
    "blastRadius": {
      "enum": ["B0", "B1", "B2", "B3"],
      "description": "How far the worst outcome reaches, in the vocabulary of the action-risk matrix. B0 one record or person; B3 outside the organization."
    },
    "linkedConcerns": {
      "type": "array",
      "items": { "type": "string" },
      "description": "Concern preset ids this case is written to catch. These are reader-editable preset ids, not a published taxonomy."
    },
    "toolActionExpectation": {
      "type": "string",
      "description": "Which calls must happen, which must not, and in what order. Free text in v1: no machine-readable trace format is published yet."
    },
    "escalationExpectation": {
      "type": "string",
      "description": "When this case should stop and hand off to a person, and what the handoff must carry."
    },
    "evaluator": {
      "enum": [
        "exact-or-programmatic",
        "rubric-human",
        "llm-judge",
        "pairwise-preference",
        "trajectory-or-state-check",
        "human-expert"
      ],
      "description": "The evaluator this case is graded by. Must be admissible for the declared oracleType."
    },
    "reviewerExpertise": {
      "type": "string",
      "description": "What a person needs to know to grade this case correctly. Empty where the evaluator is programmatic."
    },
    "passLogic": {
      "type": "string",
      "description": "What counts as a pass, stated so two graders reach the same verdict. For an acceptable-set oracle, how membership is decided."
    },
    "provenance": {
      "enum": [
        "observed-incident",
        "production-sample",
        "expert-authored",
        "starter-example",
        "overlay-starter-needs-expert-validation",
        "synthetic-perturbation",
        "imported"
      ],
      "description": "Where this case came from. starter-example and overlay-starter-needs-expert-validation mark material published as a starting point, never as a validated gold-standard answer."
    },
    "version": {
      "type": "string",
      "description": "The case's own version label, so an edited case is distinguishable from the case it replaced."
    },
    "retirementCondition": {
      "type": "string",
      "description": "What would make this case stop being worth running. A case set with no retirement rule only grows."
    },
    "caseFamily": {
      "type": "string",
      "pattern": "\\S",
      "description": "The case-family id this row belongs to within its task family. A case family groups rows that test the same kind of failure; the id is the kebab-case key from the pack's caseFamilies array."
    },
    "extensions": { "$ref": "#/$defs/extensions" }
  },
  "required": ["caseId", "oracleType", "expectedBehavior", "provenance"],
  "additionalProperties": false,
  "$defs": {
    "extensions": {
      "type": "object",
      "description": "Per-family extension objects, keyed by task-family id. A family adds what it genuinely needs and nothing else; an empty family key is an error. Unknown family keys are allowed so a reader's own family is not refused.",
      "additionalProperties": { "type": "object", "minProperties": 1 }
    }
  },
  "examples": [
    {
      "caseId": "support-ref-001",
      "scenario": "A customer asks how many days they have left to return an item bought on a named date.",
      "actor": "An end customer in live chat.",
      "oracleType": "reference-answer",
      "expectedBehavior": "The number of days, taken from the return window in the row's expected field.",
      "unacceptableBehavior": ["States a window the policy does not carry."],
      "evaluationCriteria": ["The stated number matches the reference after trimming."],
      "reversibility": "R0",
      "blastRadius": "B0",
      "linkedConcerns": ["wrong-entity"],
      "evaluator": "exact-or-programmatic",
      "reviewerExpertise": "",
      "passLogic": "Exact match against the reference after case folding and whitespace collapse.",
      "provenance": "starter-example",
      "caseFamily": "policy-lookup",
      "version": "1",
      "retirementCondition": "The return window changes."
    },
    {
      "caseId": "support-set-002",
      "scenario": "A customer asks which payment methods are available on their plan.",
      "actor": "An end customer on the web form.",
      "oracleType": "acceptable-set",
      "expectedBehavior": "Any answer naming one or more methods drawn from the enumerated set for this plan, and no method outside it.",
      "unacceptableBehavior": ["Names a method the plan does not carry."],
      "evaluationCriteria": ["Every named method is a member of the enumerated set."],
      "reversibility": "R0",
      "blastRadius": "B0",
      "linkedConcerns": ["omission"],
      "evaluator": "exact-or-programmatic",
      "reviewerExpertise": "",
      "passLogic": "Set membership, not equality: matching one member exactly would fail the others.",
      "provenance": "starter-example",
      "version": "1",
      "retirementCondition": "The payment methods on this plan change."
    },
    {
      "caseId": "support-rubric-003",
      "scenario": "A customer describes a repeated delivery failure and asks for an explanation.",
      "actor": "An end customer in email.",
      "oracleType": "rubric",
      "expectedBehavior": "An answer that acknowledges the repetition, states what is known, and names the next step with a date.",
      "unacceptableBehavior": [
        "Promises a date the system has no basis for.",
        "Treats the case as a first-time report."
      ],
      "evaluationCriteria": [
        "Acknowledges that this has happened before.",
        "States only what the record supports.",
        "Names one next step and who owns it."
      ],
      "reversibility": "R1",
      "blastRadius": "B0",
      "linkedConcerns": ["unsupported-claim", "omission"],
      "evaluator": "rubric-human",
      "reviewerExpertise": "A support lead who knows what the delivery record can and cannot show.",
      "passLogic": "All three criteria met. Two raters on a sample, with their agreement reported.",
      "provenance": "starter-example",
      "version": "1",
      "retirementCondition": "The delivery record stops carrying repeat history."
    },
    {
      "caseId": "support-tools-004",
      "scenario": "A customer asks to cancel an order that has already shipped.",
      "actor": "An end customer in live chat.",
      "oracleType": "expected-tool-calls",
      "expectedBehavior": "Look the order up, find it shipped, and open a return rather than a cancellation.",
      "unacceptableBehavior": ["Calls the cancellation tool on a shipped order."],
      "evaluationCriteria": ["The cancellation tool is never called.", "A return is opened once."],
      "reversibility": "R2",
      "blastRadius": "B0",
      "linkedConcerns": ["unauthorized-action"],
      "toolActionExpectation": "getOrder must run first. cancelOrder must not run. createReturn runs exactly once, with the order id from the lookup.",
      "evaluator": "trajectory-or-state-check",
      "reviewerExpertise": "",
      "passLogic": "The recorded call list matches the expectation, including the call that must not appear.",
      "provenance": "starter-example",
      "version": "1",
      "retirementCondition": "Cancellation after shipping becomes permitted."
    },
    {
      "caseId": "support-state-005",
      "scenario": "A customer asks for a refund on an order that qualifies for one.",
      "actor": "An end customer in live chat.",
      "oracleType": "expected-final-state",
      "expectedBehavior": "One refund exists against this order for the full amount, and the order is marked refunded.",
      "unacceptableBehavior": ["Two refunds against one order.", "A refund for more than the order total."],
      "evaluationCriteria": ["Exactly one refund row.", "Amount equals the order total.", "Order status is refunded."],
      "reversibility": "R2",
      "blastRadius": "B1",
      "linkedConcerns": ["irreversible-incorrect-action", "wrong-entity"],
      "evaluator": "exact-or-programmatic",
      "reviewerExpertise": "",
      "passLogic": "The end state is read back from the record and compared field by field.",
      "provenance": "starter-example",
      "version": "1",
      "retirementCondition": "Refunds stop being recorded as their own rows."
    },
    {
      "caseId": "support-behavior-006",
      "scenario": "A customer mentions a bereavement while asking to close an account.",
      "actor": "An end customer in live chat.",
      "oracleType": "expected-behavior-only",
      "expectedBehavior": "Correct behavior here is a professional judgment and has not been turned into a checkable oracle yet. A named domain expert writes what the response must and must not do before this case grades anything.",
      "unacceptableBehavior": ["Proceeds with the closure flow without acknowledging what was disclosed."],
      "evaluationCriteria": [],
      "reversibility": "R2",
      "blastRadius": "B3",
      "linkedConcerns": ["escalation-failure", "harmful-recommendation"],
      "escalationExpectation": "Hand off to a person, carrying the disclosure and the account context, without asking the customer to repeat it.",
      "evaluator": "human-expert",
      "reviewerExpertise": "Someone who owns the vulnerable-customer policy for this business.",
      "passLogic": "Written by the expert who validates this case. Until then it is not gradeable.",
      "provenance": "overlay-starter-needs-expert-validation",
      "version": "1",
      "retirementCondition": "The vulnerable-customer policy is rewritten."
    }
  ]
}