{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://grids365.dev/corpus/evaluation/aggregate-result.schema.json",
  "title": "Publishable aggregate Grid model evaluation result",
  "type": "object",
  "required": [
    "schemaVersion",
    "visibility",
    "split",
    "detailedResultSha256",
    "suiteManifestSha256",
    "suiteRootSha256",
    "trainingInventorySha256",
    "policySha256",
    "model",
    "runner",
    "generation",
    "execution",
    "startedAt",
    "completedAt",
    "totals",
    "metrics",
    "groups"
  ],
  "properties": {
    "schemaVersion": { "const": 1 },
    "visibility": { "const": "public-aggregate" },
    "split": { "const": "test" },
    "detailedResultSha256": { "$ref": "#/$defs/sha256" },
    "suiteManifestSha256": { "$ref": "#/$defs/sha256" },
    "suiteRootSha256": { "$ref": "#/$defs/sha256" },
    "trainingInventorySha256": { "$ref": "#/$defs/sha256" },
    "policySha256": { "$ref": "#/$defs/sha256" },
    "model": { "$ref": "#/$defs/model" },
    "runner": { "$ref": "#/$defs/runner" },
    "generation": { "$ref": "#/$defs/generation" },
    "execution": { "$ref": "#/$defs/execution" },
    "startedAt": { "type": "string", "format": "date-time" },
    "completedAt": { "type": "string", "format": "date-time" },
    "totals": { "$ref": "#/$defs/totals" },
    "metrics": { "$ref": "#/$defs/metrics" },
    "groups": {
      "type": "array",
      "maxItems": 3,
      "items": {
        "type": "object",
        "required": ["taskType", "totals", "metrics"],
        "properties": {
          "taskType": {
            "enum": ["source-generation", "diagnostic-repair", "expected-output"]
          },
          "totals": { "$ref": "#/$defs/totals" },
          "metrics": { "$ref": "#/$defs/metrics" }
        },
        "additionalProperties": false
      }
    }
  },
  "$defs": {
    "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
    "execution": {
      "type": "object",
      "required": [
        "compilerBuildProvenanceSha256",
        "runtimeBuildProvenanceSha256",
        "compilerBuildSha256",
        "runtimeBuildSha256"
      ],
      "properties": {
        "compilerBuildProvenanceSha256": { "$ref": "#/$defs/sha256" },
        "runtimeBuildProvenanceSha256": { "$ref": "#/$defs/sha256" },
        "compilerBuildSha256": { "$ref": "#/$defs/sha256" },
        "runtimeBuildSha256": { "$ref": "#/$defs/sha256" }
      },
      "additionalProperties": false
    },
    "model": {
      "type": "object",
      "required": ["id", "weightsSha256", "configurationSha256"],
      "properties": {
        "id": { "type": "string", "minLength": 1 },
        "weightsSha256": { "$ref": "#/$defs/sha256" },
        "configurationSha256": { "$ref": "#/$defs/sha256" }
      },
      "additionalProperties": false
    },
    "runner": {
      "type": "object",
      "required": ["sourceCommit", "scriptSha256"],
      "properties": {
        "sourceCommit": { "type": "string", "pattern": "^[0-9a-f]{40,64}$" },
        "scriptSha256": { "$ref": "#/$defs/sha256" }
      },
      "additionalProperties": false
    },
    "generation": {
      "type": "object",
      "required": ["temperature", "maxOutputTokens", "seed"],
      "properties": {
        "temperature": { "const": 0 },
        "maxOutputTokens": { "type": "integer", "minimum": 1, "maximum": 1048576 },
        "seed": {
          "type": "integer",
          "minimum": -2147483648,
          "maximum": 2147483647
        }
      },
      "additionalProperties": false
    },
    "totals": {
      "type": "object",
      "required": [
        "cases",
        "completed",
        "passed",
        "failed",
        "modelErrors",
        "runnerErrors"
      ],
      "properties": {
        "cases": { "type": "integer", "minimum": 5 },
        "completed": { "type": "integer", "minimum": 0 },
        "passed": { "type": "integer", "minimum": 0 },
        "failed": { "type": "integer", "minimum": 0 },
        "modelErrors": { "type": "integer", "minimum": 0 },
        "runnerErrors": { "type": "integer", "minimum": 0 }
      },
      "additionalProperties": false
    },
    "metrics": {
      "type": "object",
      "required": [
        "passRate",
        "completionRate",
        "modelErrorRate",
        "runnerErrorRate"
      ],
      "properties": {
        "passRate": { "type": "number", "minimum": 0, "maximum": 1 },
        "completionRate": { "type": "number", "minimum": 0, "maximum": 1 },
        "modelErrorRate": { "type": "number", "minimum": 0, "maximum": 1 },
        "runnerErrorRate": { "type": "number", "minimum": 0, "maximum": 1 }
      },
      "additionalProperties": false
    }
  },
  "additionalProperties": false
}