{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://latenteval.ai/schemas/evaluation-card/v1.json",
  "title": "Evaluation card, v1",
  "description": "The field set that lets a second person read one evaluation's number correctly, or run it again: what was evaluated, on which data, by which method, what came out, where it does not reach, and who ran it. Every counted field is present in a card and may be null; a null carries an entry in `unrecorded` saying why nobody recorded it, so a gap is visible rather than absent. `required` here marks which keys must be PRESENT, never which matter: whether a card is complete is a question for the reader's own tooling, not for this file. `additionalProperties` is true at every level so a card written against a later version still reads as v1-compatible. The version lives in the path. Inside a published version only additive, optional changes are allowed; a breaking change publishes v2 beside this file, never over it.",
  "x-publishedOn": "2026-08-27",
  "type": "object",
  "additionalProperties": true,
  "required": [
    "schemaVersion",
    "conformsTo",
    "externalAlignment",
    "subject",
    "task",
    "data",
    "method",
    "result",
    "limits",
    "provenance",
    "cost",
    "unrecorded"
  ],
  "properties": {
    "schemaVersion": {
      "const": "v1",
      "description": "The card's own version, not the version of the document that carries it. A v2 card is a different file at a different URL."
    },
    "conformsTo": {
      "const": "https://latenteval.ai/schemas/evaluation-card/v1.json",
      "description": "The published schema this card was written against: this file's own $id, and the only value a v1 card may carry here. A card that points anywhere else is a card for a different contract, so `const` rather than a URL pattern: the field states which published contract the card was written against, and a field that accepts any https URL states nothing."
    },
    "externalAlignment": {
      "type": "object",
      "properties": {
        "vocabulary": {
          "type": "string",
          "minLength": 1,
          "maxLength": 300
        },
        "url": {
          "type": "string",
          "pattern": "^https://",
          "maxLength": 300
        },
        "checkedOn": {
          "type": "string",
          "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
        }
      },
      "required": [
        "vocabulary",
        "url",
        "checkedOn"
      ],
      "additionalProperties": true
    },
    "subject": {
      "type": "object",
      "properties": {
        "name": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 200,
          "description": "What was evaluated."
        },
        "version": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 200,
          "description": "The pinned version or snapshot. Where a provider exposes no snapshot id, say how the subject was reached and when."
        },
        "provider": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 200
        },
        "accessMethod": {
          "type": [
            "string",
            "null"
          ],
          "enum": [
            "api",
            "local",
            "third-party",
            "unknown",
            null
          ],
          "description": "How the subject was reached. Third-party numbers and first-party numbers are not the same evidence."
        }
      },
      "required": [
        "name",
        "version",
        "provider",
        "accessMethod"
      ],
      "additionalProperties": true
    },
    "task": {
      "type": "object",
      "properties": {
        "claim": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 500,
          "description": "The one sentence this number is evidence for."
        },
        "construct": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 1000,
          "description": "What the metric stands in for."
        },
        "intendedUse": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 1000
        }
      },
      "required": [
        "claim",
        "construct",
        "intendedUse"
      ],
      "additionalProperties": true
    },
    "data": {
      "type": "object",
      "properties": {
        "source": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 1000
        },
        "version": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 200
        },
        "sha256": {
          "type": [
            "string",
            "null"
          ],
          "pattern": "^[0-9a-f]{64}$",
          "description": "SHA-256 over the bytes that were scored, 64 lowercase hex characters. A named file with no digest is a file that can change under the number."
        },
        "size": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 1,
          "description": "How many items were scored."
        },
        "split": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 300
        },
        "asOf": {
          "type": [
            "string",
            "null"
          ],
          "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
        },
        "license": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 300
        },
        "contaminationCheck": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 1000,
          "description": "What was done to check the subject had not already seen these items."
        }
      },
      "required": [
        "source",
        "version",
        "size",
        "split",
        "asOf",
        "license",
        "contaminationCheck"
      ],
      "additionalProperties": true
    },
    "method": {
      "type": "object",
      "properties": {
        "metric": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 200
        },
        "direction": {
          "type": [
            "string",
            "null"
          ],
          "enum": [
            "higher-is-better",
            "lower-is-better",
            null
          ],
          "description": "Which way is better. A metric with no direction cannot be compared."
        },
        "scale": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 200
        },
        "parser": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 1000,
          "description": "How a raw output became a score."
        },
        "runs": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 1
        },
        "temperature": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 0,
          "maximum": 2,
          "description": "Null means nobody recorded a temperature, which is not the same run as one pinned at zero."
        },
        "maxTokens": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 1
        },
        "seed": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 0,
          "description": "Zero is a real seed, not an empty field."
        },
        "otherParams": {
          "type": [
            "array",
            "null"
          ],
          "items": {
            "type": "object",
            "properties": {
              "key": {
                "type": "string",
                "minLength": 1,
                "maxLength": 120
              },
              "value": {
                "type": "string",
                "maxLength": 500
              }
            },
            "required": [
              "key",
              "value"
            ],
            "additionalProperties": true
          }
        },
        "evalPlan": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 2000,
          "description": "The scaffold or agent plan that ran."
        },
        "evalLimits": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 1000,
          "description": "The step, tool-call, wall-clock or token budget the run was held to."
        },
        "judge": {
          "type": [
            "object",
            "null"
          ],
          "description": "The model judge, where one scored this run. Null where scoring was deterministic.",
          "properties": {
            "name": {
              "type": "string",
              "minLength": 1,
              "maxLength": 200
            },
            "version": {
              "type": "string",
              "minLength": 1,
              "maxLength": 120
            },
            "promptVersion": {
              "type": [
                "string",
                "null"
              ],
              "maxLength": 120
            },
            "cardRef": {
              "type": [
                "string",
                "null"
              ],
              "pattern": "^[0-9a-f]{12}$",
              "description": "A judge card fingerprint: twelve lowercase hex characters, held as an opaque reference. A card has to read without that judge card beside it, so the readable name and version stay."
            }
          },
          "required": [
            "name",
            "version"
          ],
          "additionalProperties": true
        },
        "harness": {
          "type": [
            "object",
            "null"
          ],
          "properties": {
            "name": {
              "type": "string",
              "minLength": 1,
              "maxLength": 200
            },
            "version": {
              "type": "string",
              "minLength": 1,
              "maxLength": 120
            }
          },
          "required": [
            "name",
            "version"
          ],
          "additionalProperties": true
        }
      },
      "required": [
        "metric",
        "direction",
        "scale",
        "runs",
        "temperature",
        "maxTokens",
        "otherParams",
        "judge",
        "harness"
      ],
      "additionalProperties": true
    },
    "result": {
      "type": "object",
      "properties": {
        "headline": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 600,
          "description": "The number and what it means, in one line."
        },
        "figures": {
          "type": [
            "array",
            "null"
          ],
          "items": {
            "type": "object",
            "properties": {
              "label": {
                "type": "string",
                "minLength": 1,
                "maxLength": 200
              },
              "value": {
                "type": "string",
                "minLength": 1,
                "maxLength": 200
              },
              "interval": {
                "type": [
                  "string",
                  "null"
                ],
                "maxLength": 200
              },
              "intervalKind": {
                "type": [
                  "string",
                  "null"
                ],
                "enum": [
                  "ci",
                  "run-spread",
                  "other",
                  null
                ],
                "description": "Which kind of spread the interval is. A single-run confidence interval and run-to-run spread answer different questions."
              },
              "n": {
                "type": [
                  "integer",
                  "null"
                ],
                "minimum": 0
              }
            },
            "required": [
              "label",
              "value"
            ],
            "additionalProperties": true
          }
        },
        "envelope": {
          "type": [
            "object",
            "null"
          ],
          "additionalProperties": true,
          "description": "An optional embedded result envelope from another tool on this site, carried verbatim. Never counted: a card that restates its figures by hand is a complete card.",
          "properties": {
            "toolId": {
              "type": "string"
            },
            "schemaVersion": {
              "type": "string"
            },
            "computedAt": {
              "type": "string"
            },
            "inputs": {},
            "outputs": {},
            "reading": {
              "type": "string"
            }
          }
        }
      },
      "required": [
        "headline",
        "figures"
      ],
      "additionalProperties": true
    },
    "limits": {
      "type": [
        "array",
        "null"
      ],
      "items": {
        "type": "string",
        "minLength": 1,
        "maxLength": 1000
      },
      "description": "Where this evaluation does not reach. Limitations get written last and most often not at all, which is why a card asks for them by name."
    },
    "provenance": {
      "type": "object",
      "properties": {
        "reporter": {
          "type": [
            "string",
            "null"
          ],
          "enum": [
            "first-party",
            "third-party",
            null
          ],
          "description": "Who is reporting. First-party and third-party rows fill these fields at very different rates."
        },
        "runBy": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 200
        },
        "runOn": {
          "type": [
            "string",
            "null"
          ],
          "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
        }
      },
      "required": [
        "reporter",
        "runBy",
        "runOn"
      ],
      "additionalProperties": true
    },
    "cost": {
      "type": [
        "object",
        "null"
      ],
      "properties": {
        "amount": {
          "type": "number",
          "minimum": 0,
          "description": "Zero is a real answer."
        },
        "currency": {
          "type": "string",
          "minLength": 1,
          "maxLength": 20
        },
        "basis": {
          "type": "string",
          "minLength": 1,
          "maxLength": 300,
          "description": "What the amount bought, such as \"one full pass\"."
        }
      },
      "required": [
        "amount",
        "currency",
        "basis"
      ],
      "additionalProperties": true
    },
    "unrecorded": {
      "type": "array",
      "description": "One entry per null counted field, exactly once. The rule that every null carries one cannot be written in JSON Schema, so a reader that needs it enforces it in code.",
      "items": {
        "type": "object",
        "properties": {
          "path": {
            "type": "string",
            "minLength": 1,
            "maxLength": 120
          },
          "reason": {
            "type": "string",
            "minLength": 1,
            "maxLength": 1000
          }
        },
        "required": [
          "path",
          "reason"
        ],
        "additionalProperties": true
      }
    }
  },
  "examples": [
    {
      "schemaVersion": "v1",
      "conformsTo": "https://latenteval.ai/schemas/evaluation-card/v1.json",
      "externalAlignment": {
        "vocabulary": "Evaluation Cards, the four signals and the minimal reproducibility fields",
        "url": "https://arxiv.org/html/2606.09809v1",
        "checkedOn": "2026-08-27"
      },
      "subject": {
        "name": "Fable 5",
        "version": "As served by the API on 2026-07-07",
        "provider": "Anthropic",
        "accessMethod": "api"
      },
      "task": {
        "claim": "No measurable capability separation on short checkable tasks.",
        "construct": "Capability on short, deterministic, machine-checkable tasks.",
        "intendedUse": "Routing short checkable work between model tiers."
      },
      "data": {
        "source": "Model routing and the refusal tax: results dataset, https://latenteval.ai/benchmarks/model-routing-refusal-tax-results.csv",
        "version": "v3.withRescue",
        "sha256": "d90ed5705cb0799051110f5fa4213828c4fc1901f591753b233af0f525f693ae",
        "size": 28,
        "split": "low effort, rescue on",
        "asOf": "2026-07-07",
        "license": null,
        "contaminationCheck": null
      },
      "method": {
        "metric": "pass rate",
        "direction": "higher-is-better",
        "scale": "0 to 1",
        "parser": "Deterministic exact match, unit test, or tool-argument match. No model judge.",
        "runs": 1,
        "temperature": 1,
        "maxTokens": 8192,
        "seed": null,
        "otherParams": [
          {
            "key": "effort",
            "value": "low"
          },
          {
            "key": "server-side fallback",
            "value": "on, Fable 5 to Opus 4.8"
          }
        ],
        "evalPlan": null,
        "evalLimits": null,
        "judge": null,
        "harness": null
      },
      "result": {
        "headline": "96.4 percent with rescue on, 95 percent Wilson interval 82.3 to 99.4.",
        "figures": [
          {
            "label": "Pass rate, rescue on",
            "value": "0.9643",
            "interval": "0.8229 to 0.9937",
            "intervalKind": "ci",
            "n": 28
          },
          {
            "label": "Rescue fired",
            "value": "20 of 28 calls",
            "interval": null,
            "intervalKind": null,
            "n": 28
          },
          {
            "label": "Refusal rate, low effort, rescue off",
            "value": "21 of 28 calls",
            "interval": null,
            "intervalKind": null,
            "n": 28
          }
        ]
      },
      "limits": [
        "Short, deterministic, machine-checkable correctness only. Long-horizon agent work is unmeasured here.",
        "Small n: 14 pruned tasks, two runs each, 28 trials per arm. The intervals are wide and directional only.",
        "One provider, one task family, one 2026-07-02 classifier snapshot. The run config is not published, so the temperature and max-token values on this card cannot be recomputed from the results file."
      ],
      "provenance": {
        "reporter": "first-party",
        "runBy": "latenteval",
        "runOn": "2026-07-07"
      },
      "cost": null,
      "unrecorded": [
        {
          "path": "data.license",
          "reason": "The scored task set is not published, so it carries no license of its own. The results file is CC BY 4.0."
        },
        {
          "path": "data.contaminationCheck",
          "reason": "No contamination check was run. The tasks were written for this study rather than drawn from a public benchmark."
        },
        {
          "path": "method.seed",
          "reason": "No seed was set or recorded. The write-up notes that thinking is non-deterministic regardless."
        },
        {
          "path": "method.evalPlan",
          "reason": "There was no agent scaffold. Each task was a single call."
        },
        {
          "path": "method.evalLimits",
          "reason": "No step, tool-call or wall-clock budget applied to a single-turn call with no tools."
        },
        {
          "path": "method.judge",
          "reason": "Scoring was deterministic: exact match, unit test and tool-argument match, with no model judge."
        },
        {
          "path": "method.harness",
          "reason": "The study was pre-registered, run and scored by hand rather than through a named harness."
        },
        {
          "path": "cost",
          "reason": "Token spend for the sweep was never itemized, so no figure is recorded here."
        }
      ]
    }
  ]
}