{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://latenteval.ai/schemas/eval-set/v1.json",
  "title": "Eval set, v1",
  "description": "A vendor-neutral shape for an LLM or agent evaluation dataset: a stable id, the input, an optional expected output, optional split, slice and tag labels, and free metadata. The version lives in the path. Inside a published version only additive, optional changes are allowed; a breaking change publishes v2 beside this file, never over it.",
  "x-publishedOn": "2026-08-23",
  "oneOf": [{ "$ref": "#/$defs/wrapper" }, { "$ref": "#/$defs/rows" }],
  "examples": [
    {
      "schemaVersion": "v1",
      "rows": [
        {
          "id": "checkout-001",
          "input": "Refund the last order and email the customer.",
          "expected": "refund_issued",
          "split": "test",
          "slice": ["billing", "multi-step"],
          "tags": ["regression"],
          "metadata": { "author": "qa", "sourceTicket": 4821 }
        }
      ]
    }
  ],
  "$defs": {
    "wrapper": {
      "type": "object",
      "properties": {
        "schemaVersion": { "const": "v1" },
        "rows": { "$ref": "#/$defs/rows" }
      },
      "required": ["schemaVersion", "rows"],
      "additionalProperties": false
    },
    "rows": {
      "type": "array",
      "items": { "$ref": "#/$defs/row" }
    },
    "row": {
      "type": "object",
      "properties": {
        "id": { "type": "string", "pattern": "\\S" },
        "input": {
          "not": { "enum": [null] },
          "if": { "type": "string" },
          "then": { "pattern": "\\S" }
        },
        "expected": {},
        "split": { "type": "string" },
        "slice": { "type": "array", "items": { "type": "string" } },
        "tags": { "type": "array", "items": { "type": "string" } },
        "metadata": {
          "type": "object",
          "properties": { "evalset": false }
        }
      },
      "required": ["id", "input"],
      "additionalProperties": false
    }
  }
}