{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://grids365.dev/corpus/evaluation/detailed-result.schema.json",
  "title": "Private detailed Grid model evaluation result",
  "type": "object",
  "required": [
    "schemaVersion",
    "visibility",
    "split",
    "suiteManifestSha256",
    "suiteRootSha256",
    "trainingInventorySha256",
    "policySha256",
    "model",
    "runner",
    "generation",
    "execution",
    "startedAt",
    "completedAt",
    "cases"
  ],
  "properties": {
    "schemaVersion": { "const": 1 },
    "visibility": { "const": "private" },
    "split": { "enum": ["validation", "test"] },
    "suiteManifestSha256": { "$ref": "#/$defs/sha256" },
    "suiteRootSha256": { "$ref": "#/$defs/sha256" },
    "trainingInventorySha256": { "$ref": "#/$defs/sha256" },
    "policySha256": { "$ref": "#/$defs/sha256" },
    "model": { "$ref": "#/$defs/model" },
    "runner": { "$ref": "#/$defs/runner" },
    "generation": { "$ref": "#/$defs/generation" },
    "execution": { "$ref": "#/$defs/execution" },
    "startedAt": { "type": "string", "format": "date-time" },
    "completedAt": { "type": "string", "format": "date-time" },
    "cases": {
      "type": "array",
      "minItems": 1,
      "items": {
        "type": "object",
        "required": ["caseId", "status", "candidateOutput", "scores", "diagnostics"],
        "properties": {
          "caseId": { "$ref": "#/$defs/opaqueId" },
          "status": { "enum": ["completed", "model-error", "runner-error"] },
          "candidateOutput": { "type": "string", "maxLength": 16777216 },
          "scores": {
            "type": "array",
            "items": {
              "type": "object",
              "required": ["scorer", "passed", "value"],
              "properties": {
                "scorer": { "type": "string", "minLength": 1 },
                "passed": { "type": "boolean" },
                "value": { "type": "boolean" }
              },
              "additionalProperties": false
            }
          },
          "diagnostics": {
            "type": "array",
            "items": {
              "type": "object",
              "required": ["code", "message"],
              "properties": {
                "code": { "type": "string", "minLength": 1 },
                "message": { "type": "string" },
                "span": {
                  "type": "object",
                  "required": ["start", "end"],
                  "properties": {
                    "start": { "type": "integer", "minimum": 0 },
                    "end": { "type": "integer", "minimum": 0 }
                  },
                  "additionalProperties": false
                }
              },
              "additionalProperties": false
            }
          }
        },
        "additionalProperties": false
      }
    }
  },
  "$defs": {
    "opaqueId": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
    "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
    "execution": {
      "type": "object",
      "required": [
        "compilerBuildProvenanceSha256",
        "runtimeBuildProvenanceSha256",
        "compilerBuildSha256",
        "runtimeBuildSha256"
      ],
      "properties": {
        "compilerBuildProvenanceSha256": { "$ref": "#/$defs/sha256" },
        "runtimeBuildProvenanceSha256": { "$ref": "#/$defs/sha256" },
        "compilerBuildSha256": { "$ref": "#/$defs/sha256" },
        "runtimeBuildSha256": { "$ref": "#/$defs/sha256" }
      },
      "additionalProperties": false
    },
    "model": {
      "type": "object",
      "required": ["id", "weightsSha256", "configurationSha256"],
      "properties": {
        "id": { "type": "string", "minLength": 1 },
        "weightsSha256": { "$ref": "#/$defs/sha256" },
        "configurationSha256": { "$ref": "#/$defs/sha256" }
      },
      "additionalProperties": false
    },
    "runner": {
      "type": "object",
      "required": ["sourceCommit", "scriptSha256"],
      "properties": {
        "sourceCommit": { "type": "string", "pattern": "^[0-9a-f]{40,64}$" },
        "scriptSha256": { "$ref": "#/$defs/sha256" }
      },
      "additionalProperties": false
    },
    "generation": {
      "type": "object",
      "required": ["temperature", "maxOutputTokens", "seed"],
      "properties": {
        "temperature": { "const": 0 },
        "maxOutputTokens": { "type": "integer", "minimum": 1, "maximum": 1048576 },
        "seed": {
          "type": "integer",
          "minimum": -2147483648,
          "maximum": 2147483647
        }
      },
      "additionalProperties": false
    }
  },
  "additionalProperties": false
}