{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://github.com/andrewhuot/headless-chat-sdk/schemas/eval-config.schema.json",
  "title": "Eval Config",
  "definitions": {
    "EvalConfig": {
      "type": "object",
      "properties": {
        "providers": {
          "type": "object",
          "properties": {
            "anthropic": { "type": "object", "properties": { "model": { "type": "string" } } },
            "openai": { "type": "object", "properties": { "model": { "type": "string" } } },
            "gemini": { "type": "object", "properties": { "model": { "type": "string" } } }
          }
        },
        "regressionThresholds": {
          "type": "object",
          "properties": {
            "deflectionDropPp": { "type": "number" },
            "p95LatencyIncreasePct": { "type": "number" },
            "ttftP95IncreasePct": { "type": "number" },
            "passRateDropPp": { "type": "number" },
            "hallucinationRateIncreasePp": { "type": "number" }
          }
        },
        "scorers": {
          "type": "object",
          "additionalProperties": {
            "type": "object",
            "properties": { "enabled": { "type": "boolean" } }
          }
        }
      }
    }
  }
}