Python (SDK)

from meetkai_mka1 import SDK


with SDK(
    bearer_auth="<YOUR_BEARER_TOKEN_HERE>",
) as sdk:

    res = sdk.llm.evals.append_agent_trials(run_id="eval_run_aa87e2b1112a455b8deabed784372198", trials=[
        {
            "task_id": "fix-permissions",
            "sample_index": 0,
            "prompt": "The test suite fails because the fixture directory is not writable. Fix it.",
            "target": "all tests pass",
            "output_text": "9 passed in 2.25s",
            "scores": {
                "reward": 1,
                "resolved": 1,
            },
            "cost_usd": 0.0279,
            "input_tokens": 658456,
            "output_tokens": 6489,
            "cached_tokens": 658366,
        },
    ], finalize=False)

    # Handle response
    print(res)

import { SDK } from "@meetkai/mka1";

const sdk = new SDK({
  bearerAuth: "<YOUR_BEARER_TOKEN_HERE>",
});

async function run() {
  const result = await sdk.llm.evals.appendAgentTrials({
    runId: "eval_run_aa87e2b1112a455b8deabed784372198",
    appendAgentTrialsRequest: {
      trials: [
        {
          taskId: "fix-permissions",
          sampleIndex: 0,
          prompt: "The test suite fails because the fixture directory is not writable. Fix it.",
          target: "all tests pass",
          outputText: "9 passed in 2.25s",
          scores: {
            "reward": 1,
            "resolved": 1,
          },
          costUsd: 0.0279,
          inputTokens: 658456,
          outputTokens: 6489,
          cachedTokens: 658366,
        },
      ],
    },
  });

  console.log(result);
}

run();

using MeetKai.MKA1;
using MeetKai.MKA1.Types.Components;
using System.Collections.Generic;

var sdk = new SDK(bearerAuth: "<YOUR_BEARER_TOKEN_HERE>");

var res = await sdk.Llm.Evals.AppendAgentTrialsAsync(
    runId: "eval_run_aa87e2b1112a455b8deabed784372198",
    body: new AppendAgentTrialsRequest() {
        Trials = new List<AppendAgentTrialsRequestTrial>() {
            new AppendAgentTrialsRequestTrial() {
                TaskId = "fix-permissions",
                SampleIndex = 0,
                Prompt = "The test suite fails because the fixture directory is not writable. Fix it.",
                Target = "all tests pass",
                OutputText = "9 passed in 2.25s",
                Scores = new Dictionary<string, double>() {
                    { "reward", 1D },
                    { "resolved", 1D },
                },
                CostUsd = 0.0279D,
                InputTokens = 658456,
                OutputTokens = 6489,
                CachedTokens = 658366,
            },
        },
    }
);

// handle response

curl --request POST \
  --url https://apigw.mka1.com/api/v1/llm/evals/runs/{run_id}/trials \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "trials": [
    {
      "task_id": "fix-permissions",
      "sample_index": 0,
      "status": "completed",
      "prompt": "The test suite fails because the fixture directory is not writable. Fix it.",
      "target": "all tests pass",
      "output_text": "9 passed in 2.25s",
      "scores": {
        "reward": 1,
        "resolved": 1
      },
      "cost_usd": 0.0279,
      "input_tokens": 658456,
      "output_tokens": 6489,
      "cached_tokens": 658366
    }
  ],
  "finalize": false
}
'

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    trials: [
      {
        task_id: 'fix-permissions',
        sample_index: 0,
        status: 'completed',
        prompt: 'The test suite fails because the fixture directory is not writable. Fix it.',
        target: 'all tests pass',
        output_text: '9 passed in 2.25s',
        scores: {reward: 1, resolved: 1},
        cost_usd: 0.0279,
        input_tokens: 658456,
        output_tokens: 6489,
        cached_tokens: 658366
      }
    ],
    finalize: false
  })
};

fetch('https://apigw.mka1.com/api/v1/llm/evals/runs/{run_id}/trials', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://apigw.mka1.com/api/v1/llm/evals/runs/{run_id}/trials",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'trials' => [
        [
                'task_id' => 'fix-permissions',
                'sample_index' => 0,
                'status' => 'completed',
                'prompt' => 'The test suite fails because the fixture directory is not writable. Fix it.',
                'target' => 'all tests pass',
                'output_text' => '9 passed in 2.25s',
                'scores' => [
                                'reward' => 1,
                                'resolved' => 1
                ],
                'cost_usd' => 0.0279,
                'input_tokens' => 658456,
                'output_tokens' => 6489,
                'cached_tokens' => 658366
        ]
    ],
    'finalize' => false
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://apigw.mka1.com/api/v1/llm/evals/runs/{run_id}/trials"

	payload := strings.NewReader("{\n  \"trials\": [\n    {\n      \"task_id\": \"fix-permissions\",\n      \"sample_index\": 0,\n      \"status\": \"completed\",\n      \"prompt\": \"The test suite fails because the fixture directory is not writable. Fix it.\",\n      \"target\": \"all tests pass\",\n      \"output_text\": \"9 passed in 2.25s\",\n      \"scores\": {\n        \"reward\": 1,\n        \"resolved\": 1\n      },\n      \"cost_usd\": 0.0279,\n      \"input_tokens\": 658456,\n      \"output_tokens\": 6489,\n      \"cached_tokens\": 658366\n    }\n  ],\n  \"finalize\": false\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://apigw.mka1.com/api/v1/llm/evals/runs/{run_id}/trials")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"trials\": [\n    {\n      \"task_id\": \"fix-permissions\",\n      \"sample_index\": 0,\n      \"status\": \"completed\",\n      \"prompt\": \"The test suite fails because the fixture directory is not writable. Fix it.\",\n      \"target\": \"all tests pass\",\n      \"output_text\": \"9 passed in 2.25s\",\n      \"scores\": {\n        \"reward\": 1,\n        \"resolved\": 1\n      },\n      \"cost_usd\": 0.0279,\n      \"input_tokens\": 658456,\n      \"output_tokens\": 6489,\n      \"cached_tokens\": 658366\n    }\n  ],\n  \"finalize\": false\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://apigw.mka1.com/api/v1/llm/evals/runs/{run_id}/trials")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"trials\": [\n    {\n      \"task_id\": \"fix-permissions\",\n      \"sample_index\": 0,\n      \"status\": \"completed\",\n      \"prompt\": \"The test suite fails because the fixture directory is not writable. Fix it.\",\n      \"target\": \"all tests pass\",\n      \"output_text\": \"9 passed in 2.25s\",\n      \"scores\": {\n        \"reward\": 1,\n        \"resolved\": 1\n      },\n      \"cost_usd\": 0.0279,\n      \"input_tokens\": 658456,\n      \"output_tokens\": 6489,\n      \"cached_tokens\": 658366\n    }\n  ],\n  \"finalize\": false\n}"

response = http.request(request)
puts response.read_body

{
  "id": "eval_run_aa87e2b1112a455b8deabed784372198",
  "object": "eval.run",
  "suite_id": "eval_suite_aa87e2b1112a455b8deabed784372198",
  "suite_version": 1,
  "suite_version_id": "eval_sver_aa87e2b1112a455b8deabed784372198",
  "org_id": "org-acme",
  "team_id": "team-research",
  "status": "in_progress",
  "models": [
    "auto"
  ],
  "task_ids": null,
  "judge_model": "auto",
  "embedding_model": "auto",
  "generation": {
    "temperature": 0,
    "max_output_tokens": 512
  },
  "request_counts": {
    "total": 100,
    "completed": 10,
    "failed": 0
  },
  "metrics": null,
  "error": null,
  "artifact_file_ids": [],
  "metadata": {
    "purpose": "mvp"
  },
  "agent_name": "omp",
  "agent_version": null,
  "agent_effort": null,
  "cost_usd": null,
  "created_at": 1704067200,
  "started_at": 1704067210,
  "completed_at": null,
  "cancelled_at": null,
  "failed_at": null
}

Evals

Append trials to an open coding-agent run

Adds finished trials to a run opened with finalize: false, so a job started from MKA1 can be watched while it runs instead of appearing complete out of nowhere at the end. Each trial pins its own sample_index and upserts on (task, model, sample_index): a harness cannot tell a timeout from a lost response, so re-sending a batch is safe and replaces rather than duplicates. The whole batch and its aggregate recompute land in one transaction, serialized per run. Counts, metrics and cost are recomputed from the stored rows on every call, never accumulated from the request, so a replayed batch cannot double them. Send finalize: true (usually with an empty trials) to close the run — until then it stays in_progress and off the leaderboards, which only count completed runs. Appending to an already-completed or cancelled run is rejected: that record is finished, and rewriting it would silently move a number someone may have already read.

POST

api

llm

evals

runs

{run_id}

trials

Python (SDK)

from meetkai_mka1 import SDK


with SDK(
    bearer_auth="<YOUR_BEARER_TOKEN_HERE>",
) as sdk:

    res = sdk.llm.evals.append_agent_trials(run_id="eval_run_aa87e2b1112a455b8deabed784372198", trials=[
        {
            "task_id": "fix-permissions",
            "sample_index": 0,
            "prompt": "The test suite fails because the fixture directory is not writable. Fix it.",
            "target": "all tests pass",
            "output_text": "9 passed in 2.25s",
            "scores": {
                "reward": 1,
                "resolved": 1,
            },
            "cost_usd": 0.0279,
            "input_tokens": 658456,
            "output_tokens": 6489,
            "cached_tokens": 658366,
        },
    ], finalize=False)

    # Handle response
    print(res)

import { SDK } from "@meetkai/mka1";

const sdk = new SDK({
  bearerAuth: "<YOUR_BEARER_TOKEN_HERE>",
});

async function run() {
  const result = await sdk.llm.evals.appendAgentTrials({
    runId: "eval_run_aa87e2b1112a455b8deabed784372198",
    appendAgentTrialsRequest: {
      trials: [
        {
          taskId: "fix-permissions",
          sampleIndex: 0,
          prompt: "The test suite fails because the fixture directory is not writable. Fix it.",
          target: "all tests pass",
          outputText: "9 passed in 2.25s",
          scores: {
            "reward": 1,
            "resolved": 1,
          },
          costUsd: 0.0279,
          inputTokens: 658456,
          outputTokens: 6489,
          cachedTokens: 658366,
        },
      ],
    },
  });

  console.log(result);
}

run();

using MeetKai.MKA1;
using MeetKai.MKA1.Types.Components;
using System.Collections.Generic;

var sdk = new SDK(bearerAuth: "<YOUR_BEARER_TOKEN_HERE>");

var res = await sdk.Llm.Evals.AppendAgentTrialsAsync(
    runId: "eval_run_aa87e2b1112a455b8deabed784372198",
    body: new AppendAgentTrialsRequest() {
        Trials = new List<AppendAgentTrialsRequestTrial>() {
            new AppendAgentTrialsRequestTrial() {
                TaskId = "fix-permissions",
                SampleIndex = 0,
                Prompt = "The test suite fails because the fixture directory is not writable. Fix it.",
                Target = "all tests pass",
                OutputText = "9 passed in 2.25s",
                Scores = new Dictionary<string, double>() {
                    { "reward", 1D },
                    { "resolved", 1D },
                },
                CostUsd = 0.0279D,
                InputTokens = 658456,
                OutputTokens = 6489,
                CachedTokens = 658366,
            },
        },
    }
);

// handle response

curl --request POST \
  --url https://apigw.mka1.com/api/v1/llm/evals/runs/{run_id}/trials \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "trials": [
    {
      "task_id": "fix-permissions",
      "sample_index": 0,
      "status": "completed",
      "prompt": "The test suite fails because the fixture directory is not writable. Fix it.",
      "target": "all tests pass",
      "output_text": "9 passed in 2.25s",
      "scores": {
        "reward": 1,
        "resolved": 1
      },
      "cost_usd": 0.0279,
      "input_tokens": 658456,
      "output_tokens": 6489,
      "cached_tokens": 658366
    }
  ],
  "finalize": false
}
'

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    trials: [
      {
        task_id: 'fix-permissions',
        sample_index: 0,
        status: 'completed',
        prompt: 'The test suite fails because the fixture directory is not writable. Fix it.',
        target: 'all tests pass',
        output_text: '9 passed in 2.25s',
        scores: {reward: 1, resolved: 1},
        cost_usd: 0.0279,
        input_tokens: 658456,
        output_tokens: 6489,
        cached_tokens: 658366
      }
    ],
    finalize: false
  })
};

fetch('https://apigw.mka1.com/api/v1/llm/evals/runs/{run_id}/trials', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://apigw.mka1.com/api/v1/llm/evals/runs/{run_id}/trials",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'trials' => [
        [
                'task_id' => 'fix-permissions',
                'sample_index' => 0,
                'status' => 'completed',
                'prompt' => 'The test suite fails because the fixture directory is not writable. Fix it.',
                'target' => 'all tests pass',
                'output_text' => '9 passed in 2.25s',
                'scores' => [
                                'reward' => 1,
                                'resolved' => 1
                ],
                'cost_usd' => 0.0279,
                'input_tokens' => 658456,
                'output_tokens' => 6489,
                'cached_tokens' => 658366
        ]
    ],
    'finalize' => false
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://apigw.mka1.com/api/v1/llm/evals/runs/{run_id}/trials"

	payload := strings.NewReader("{\n  \"trials\": [\n    {\n      \"task_id\": \"fix-permissions\",\n      \"sample_index\": 0,\n      \"status\": \"completed\",\n      \"prompt\": \"The test suite fails because the fixture directory is not writable. Fix it.\",\n      \"target\": \"all tests pass\",\n      \"output_text\": \"9 passed in 2.25s\",\n      \"scores\": {\n        \"reward\": 1,\n        \"resolved\": 1\n      },\n      \"cost_usd\": 0.0279,\n      \"input_tokens\": 658456,\n      \"output_tokens\": 6489,\n      \"cached_tokens\": 658366\n    }\n  ],\n  \"finalize\": false\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://apigw.mka1.com/api/v1/llm/evals/runs/{run_id}/trials")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"trials\": [\n    {\n      \"task_id\": \"fix-permissions\",\n      \"sample_index\": 0,\n      \"status\": \"completed\",\n      \"prompt\": \"The test suite fails because the fixture directory is not writable. Fix it.\",\n      \"target\": \"all tests pass\",\n      \"output_text\": \"9 passed in 2.25s\",\n      \"scores\": {\n        \"reward\": 1,\n        \"resolved\": 1\n      },\n      \"cost_usd\": 0.0279,\n      \"input_tokens\": 658456,\n      \"output_tokens\": 6489,\n      \"cached_tokens\": 658366\n    }\n  ],\n  \"finalize\": false\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://apigw.mka1.com/api/v1/llm/evals/runs/{run_id}/trials")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"trials\": [\n    {\n      \"task_id\": \"fix-permissions\",\n      \"sample_index\": 0,\n      \"status\": \"completed\",\n      \"prompt\": \"The test suite fails because the fixture directory is not writable. Fix it.\",\n      \"target\": \"all tests pass\",\n      \"output_text\": \"9 passed in 2.25s\",\n      \"scores\": {\n        \"reward\": 1,\n        \"resolved\": 1\n      },\n      \"cost_usd\": 0.0279,\n      \"input_tokens\": 658456,\n      \"output_tokens\": 6489,\n      \"cached_tokens\": 658366\n    }\n  ],\n  \"finalize\": false\n}"

response = http.request(request)
puts response.read_body

{
  "id": "eval_run_aa87e2b1112a455b8deabed784372198",
  "object": "eval.run",
  "suite_id": "eval_suite_aa87e2b1112a455b8deabed784372198",
  "suite_version": 1,
  "suite_version_id": "eval_sver_aa87e2b1112a455b8deabed784372198",
  "org_id": "org-acme",
  "team_id": "team-research",
  "status": "in_progress",
  "models": [
    "auto"
  ],
  "task_ids": null,
  "judge_model": "auto",
  "embedding_model": "auto",
  "generation": {
    "temperature": 0,
    "max_output_tokens": 512
  },
  "request_counts": {
    "total": 100,
    "completed": 10,
    "failed": 0
  },
  "metrics": null,
  "error": null,
  "artifact_file_ids": [],
  "metadata": {
    "purpose": "mvp"
  },
  "agent_name": "omp",
  "agent_version": null,
  "agent_effort": null,
  "cost_usd": null,
  "created_at": 1704067200,
  "started_at": 1704067210,
  "completed_at": null,
  "cancelled_at": null,
  "failed_at": null
}

Authorizations

Authorization

string

header

required

Gateway auth: send Authorization: Bearer <mka1-api-key>. For multi-user server-side integrations, you can also send X-On-Behalf-Of: <external-user-id>.

Headers

X-On-Behalf-Of

string

Optional external end-user identifier forwarded by the API gateway.

Path Parameters

run_id

string

required

Body

application/json

trials

object[]

Each trial pins its own sample_index (unique per task and model): re-sending a batch replaces the same rows instead of duplicating them, so a retried batch is safe.

Maximum array length: 1000

Show child attributes

finalize

boolean

default:false

Marks the run completed once these trials land — the last call of a streaming job sends this, usually with an empty trials.

Response

200 - application/json

string

required

object

any

required

suite_id

string

required

suite_version

integer

required

Required range: -9007199254740991 <= x <= 9007199254740991

suite_version_id

string

required

org_id

string

required

The org that owns this run.

team_id

string

required

The team that owns this run.

status

enum<string>

required

Available options:

queued,

in_progress,

finalizing,

completed,

failed,

cancelling,

cancelled

models

string[]

required

task_ids

string[] | null

required

judge_model

string | null

required

embedding_model

string | null

required

generation

object

required

Show child attributes

request_counts

object

required

Show child attributes

metrics

object | null

required

Show child attributes

error

object | null

required

Show child attributes

artifact_file_ids

string[]

required

metadata

object

required

Show child attributes

agent_name

string | null

required

Coding-agent harness that produced this run (e.g. omp, claude-code). Null for every non-harness eval kind — a null here is not a missing value, it means the run is not an agent run.

agent_version

string | null

required

Version of agent_name, as the harness reported it. Null when agent_name is.

agent_effort

string | null

required

Reasoning-effort setting the agent ran at (e.g. medium, xhigh). Part of the leaderboard's row identity: the same agent and model at two efforts are two rows, not one. Null when agent_name is.

cost_usd

number | null

required

Total spend for the run in USD. Null when the harness did not report cost.

created_at

integer

required

Required range: -9007199254740991 <= x <= 9007199254740991

started_at

integer | null

required

Required range: -9007199254740991 <= x <= 9007199254740991

completed_at

integer | null

required

Required range: -9007199254740991 <= x <= 9007199254740991

cancelled_at

integer | null

required

Required range: -9007199254740991 <= x <= 9007199254740991

failed_at

integer | null

required

Required range: -9007199254740991 <= x <= 9007199254740991

⌘I

Overview

Responses API

Agents API

Platform APIs

Administration

Sandbox API

Compute API

Repositories API

Deprecated

Other

Append trials to an open coding-agent run

Authorizations

Headers

Path Parameters

Body

Response