{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://github.com/andrewhuot/headless-chat-sdk/schemas/eval-report.schema.json",
  "title": "Eval Report",
  "definitions": {
    "EvalReport": {
      "type": "object",
      "required": ["runId", "startedAt", "finishedAt", "scenarios", "metrics", "env"],
      "properties": {
        "runId": { "type": "string" },
        "startedAt": { "type": "string", "format": "date-time" },
        "finishedAt": { "type": "string", "format": "date-time" },
        "scenarios": { "type": "array", "items": { "$ref": "#/definitions/ScenarioResult" } },
        "metrics": { "$ref": "#/definitions/AggregateMetrics" },
        "env": { "$ref": "#/definitions/EvalReportEnv" }
      }
    },
    "AggregateMetrics": {
      "type": "object",
      "required": [
        "totalScenarios",
        "passedScenarios",
        "failedScenarios",
        "skippedScenarios",
        "passRate",
        "deflectionRate",
        "latencyP50Ms",
        "latencyP95Ms",
        "ttftP50Ms",
        "ttftP95Ms",
        "hallucinationRate",
        "toolCallAccuracy"
      ],
      "properties": {
        "totalScenarios": { "type": "number" },
        "passedScenarios": { "type": "number" },
        "failedScenarios": { "type": "number" },
        "skippedScenarios": { "type": "number" },
        "passRate": { "type": "number" },
        "deflectionRate": { "type": "number" },
        "latencyP50Ms": { "type": "number" },
        "latencyP95Ms": { "type": "number" },
        "ttftP50Ms": { "type": "number" },
        "ttftP95Ms": { "type": "number" },
        "hallucinationRate": { "type": "number" },
        "toolCallAccuracy": { "type": "number" }
      }
    },
    "ScenarioResult": {
      "type": "object",
      "required": ["scenarioId", "name", "status", "expectations"],
      "properties": {
        "scenarioId": { "type": "string" },
        "name": { "type": "string" },
        "tags": { "type": "array", "items": { "type": "string" } },
        "status": { "type": "string", "enum": ["passed", "failed", "skipped"] },
        "expectations": { "type": "array" }
      }
    },
    "EvalReportEnv": {
      "type": "object",
      "required": ["node", "sdkVersion", "providers"],
      "properties": {
        "node": { "type": "string" },
        "sdkVersion": { "type": "string" },
        "providers": { "type": "object" }
      }
    }
  }
}