Python (SDK)
from meetkai_mka1 import SDK
with SDK(
bearer_auth="<YOUR_BEARER_TOKEN_HERE>",
) as sdk:
res = sdk.llm.evals.create_run(suite_id="eval_suite_aa87e2b1112a455b8deabed784372198", models=[
"auto",
], judge_model="auto", embedding_model="auto", generation={
"temperature": 0.0,
"max_gen_toks": 512,
"until": [
"<|endoftext|>",
],
"do_sample": False,
"chat_template_kwargs": {
"enable_thinking": False,
},
"timeout_seconds": 120,
"max_retries": 2,
"max_empty_retries": 1,
}, generation_concurrency=4, grader_concurrency=2, max_workflow_sample_activities=5000, metadata={
"purpose": "mvp",
})
# Handle response
print(res)import { SDK } from "@meetkai/mka1";
const sdk = new SDK({
bearerAuth: "<YOUR_BEARER_TOKEN_HERE>",
});
async function run() {
const result = await sdk.llm.evals.createRun({
createEvalRunRequest: {
suiteId: "eval_suite_aa87e2b1112a455b8deabed784372198",
models: [
"auto",
],
judgeModel: "auto",
embeddingModel: "auto",
generation: {
temperature: 0,
maxGenToks: 512,
until: [
"<|endoftext|>",
],
doSample: false,
chatTemplateKwargs: {
"enable_thinking": false,
},
timeoutSeconds: 120,
maxRetries: 2,
maxEmptyRetries: 1,
},
generationConcurrency: 4,
graderConcurrency: 2,
maxWorkflowSampleActivities: 5000,
metadata: {
"purpose": "mvp",
},
},
});
console.log(result);
}
run();using MeetKai.MKA1;
using MeetKai.MKA1.Types.Components;
using System.Collections.Generic;
var sdk = new SDK(bearerAuth: "<YOUR_BEARER_TOKEN_HERE>");
var res = await sdk.Llm.Evals.CreateRunAsync(body: new MeetKai.MKA1.Types.Components.CreateEvalRunRequest() {
SuiteId = "eval_suite_aa87e2b1112a455b8deabed784372198",
Models = new List<string>() {
"auto",
},
JudgeModel = "auto",
EmbeddingModel = "auto",
Generation = new EvalGenerationRequestConfig() {
Temperature = 0D,
MaxGenToks = 512,
Until = new List<string>() {
"<|endoftext|>",
},
DoSample = false,
ChatTemplateKwargs = new Dictionary<string, object>() {
{ "enable_thinking", false },
},
TimeoutSeconds = 120,
MaxRetries = 2,
MaxEmptyRetries = 1,
},
GenerationConcurrency = 4,
GraderConcurrency = 2,
MaxWorkflowSampleActivities = 5000,
Metadata = new Dictionary<string, string>() {
{ "purpose", "mvp" },
},
});
// handle responsecurl --request POST \
--url https://apigw.mka1.com/api/v1/llm/evals/runs \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"suite_id": "eval_suite_aa87e2b1112a455b8deabed784372198",
"models": [
"auto"
],
"judge_model": "auto",
"embedding_model": "auto",
"generation": {
"temperature": 0,
"max_gen_toks": 512,
"until": [
"<|endoftext|>"
],
"do_sample": false,
"chat_template_kwargs": {
"enable_thinking": false
},
"max_retries": 2,
"max_empty_retries": 1,
"timeout_seconds": 120
},
"generation_concurrency": 4,
"grader_concurrency": 2,
"max_workflow_sample_activities": 5000,
"metadata": {
"purpose": "mvp"
}
}
'const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
suite_id: 'eval_suite_aa87e2b1112a455b8deabed784372198',
models: ['auto'],
judge_model: 'auto',
embedding_model: 'auto',
generation: {
temperature: 0,
max_gen_toks: 512,
until: ['<|endoftext|>'],
do_sample: false,
chat_template_kwargs: {enable_thinking: false},
max_retries: 2,
max_empty_retries: 1,
timeout_seconds: 120
},
generation_concurrency: 4,
grader_concurrency: 2,
max_workflow_sample_activities: 5000,
metadata: {purpose: 'mvp'}
})
};
fetch('https://apigw.mka1.com/api/v1/llm/evals/runs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://apigw.mka1.com/api/v1/llm/evals/runs",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'suite_id' => 'eval_suite_aa87e2b1112a455b8deabed784372198',
'models' => [
'auto'
],
'judge_model' => 'auto',
'embedding_model' => 'auto',
'generation' => [
'temperature' => 0,
'max_gen_toks' => 512,
'until' => [
'<|endoftext|>'
],
'do_sample' => false,
'chat_template_kwargs' => [
'enable_thinking' => false
],
'max_retries' => 2,
'max_empty_retries' => 1,
'timeout_seconds' => 120
],
'generation_concurrency' => 4,
'grader_concurrency' => 2,
'max_workflow_sample_activities' => 5000,
'metadata' => [
'purpose' => 'mvp'
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://apigw.mka1.com/api/v1/llm/evals/runs"
payload := strings.NewReader("{\n \"suite_id\": \"eval_suite_aa87e2b1112a455b8deabed784372198\",\n \"models\": [\n \"auto\"\n ],\n \"judge_model\": \"auto\",\n \"embedding_model\": \"auto\",\n \"generation\": {\n \"temperature\": 0,\n \"max_gen_toks\": 512,\n \"until\": [\n \"<|endoftext|>\"\n ],\n \"do_sample\": false,\n \"chat_template_kwargs\": {\n \"enable_thinking\": false\n },\n \"max_retries\": 2,\n \"max_empty_retries\": 1,\n \"timeout_seconds\": 120\n },\n \"generation_concurrency\": 4,\n \"grader_concurrency\": 2,\n \"max_workflow_sample_activities\": 5000,\n \"metadata\": {\n \"purpose\": \"mvp\"\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://apigw.mka1.com/api/v1/llm/evals/runs")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"suite_id\": \"eval_suite_aa87e2b1112a455b8deabed784372198\",\n \"models\": [\n \"auto\"\n ],\n \"judge_model\": \"auto\",\n \"embedding_model\": \"auto\",\n \"generation\": {\n \"temperature\": 0,\n \"max_gen_toks\": 512,\n \"until\": [\n \"<|endoftext|>\"\n ],\n \"do_sample\": false,\n \"chat_template_kwargs\": {\n \"enable_thinking\": false\n },\n \"max_retries\": 2,\n \"max_empty_retries\": 1,\n \"timeout_seconds\": 120\n },\n \"generation_concurrency\": 4,\n \"grader_concurrency\": 2,\n \"max_workflow_sample_activities\": 5000,\n \"metadata\": {\n \"purpose\": \"mvp\"\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://apigw.mka1.com/api/v1/llm/evals/runs")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"suite_id\": \"eval_suite_aa87e2b1112a455b8deabed784372198\",\n \"models\": [\n \"auto\"\n ],\n \"judge_model\": \"auto\",\n \"embedding_model\": \"auto\",\n \"generation\": {\n \"temperature\": 0,\n \"max_gen_toks\": 512,\n \"until\": [\n \"<|endoftext|>\"\n ],\n \"do_sample\": false,\n \"chat_template_kwargs\": {\n \"enable_thinking\": false\n },\n \"max_retries\": 2,\n \"max_empty_retries\": 1,\n \"timeout_seconds\": 120\n },\n \"generation_concurrency\": 4,\n \"grader_concurrency\": 2,\n \"max_workflow_sample_activities\": 5000,\n \"metadata\": {\n \"purpose\": \"mvp\"\n }\n}"
response = http.request(request)
puts response.read_body{
"id": "eval_run_aa87e2b1112a455b8deabed784372198",
"object": "eval.run",
"suite_id": "eval_suite_aa87e2b1112a455b8deabed784372198",
"suite_version": 1,
"suite_version_id": "eval_sver_aa87e2b1112a455b8deabed784372198",
"org_id": "org-acme",
"team_id": "team-research",
"status": "in_progress",
"models": [
"auto"
],
"task_ids": null,
"judge_model": "auto",
"embedding_model": "auto",
"generation": {
"temperature": 0,
"max_output_tokens": 512
},
"request_counts": {
"total": 100,
"completed": 10,
"failed": 0
},
"metrics": null,
"error": null,
"artifact_file_ids": [],
"metadata": {
"purpose": "mvp"
},
"agent_name": null,
"agent_version": null,
"agent_effort": null,
"cost_usd": null,
"created_at": 1704067200,
"started_at": 1704067210,
"completed_at": null,
"cancelled_at": null,
"failed_at": null
}Evals
Create an eval run
Starts a durable eval run over the selected suite version, tasks, and models.
POST
/
api
/
v1
/
llm
/
evals
/
runs
Python (SDK)
from meetkai_mka1 import SDK
with SDK(
bearer_auth="<YOUR_BEARER_TOKEN_HERE>",
) as sdk:
res = sdk.llm.evals.create_run(suite_id="eval_suite_aa87e2b1112a455b8deabed784372198", models=[
"auto",
], judge_model="auto", embedding_model="auto", generation={
"temperature": 0.0,
"max_gen_toks": 512,
"until": [
"<|endoftext|>",
],
"do_sample": False,
"chat_template_kwargs": {
"enable_thinking": False,
},
"timeout_seconds": 120,
"max_retries": 2,
"max_empty_retries": 1,
}, generation_concurrency=4, grader_concurrency=2, max_workflow_sample_activities=5000, metadata={
"purpose": "mvp",
})
# Handle response
print(res)import { SDK } from "@meetkai/mka1";
const sdk = new SDK({
bearerAuth: "<YOUR_BEARER_TOKEN_HERE>",
});
async function run() {
const result = await sdk.llm.evals.createRun({
createEvalRunRequest: {
suiteId: "eval_suite_aa87e2b1112a455b8deabed784372198",
models: [
"auto",
],
judgeModel: "auto",
embeddingModel: "auto",
generation: {
temperature: 0,
maxGenToks: 512,
until: [
"<|endoftext|>",
],
doSample: false,
chatTemplateKwargs: {
"enable_thinking": false,
},
timeoutSeconds: 120,
maxRetries: 2,
maxEmptyRetries: 1,
},
generationConcurrency: 4,
graderConcurrency: 2,
maxWorkflowSampleActivities: 5000,
metadata: {
"purpose": "mvp",
},
},
});
console.log(result);
}
run();using MeetKai.MKA1;
using MeetKai.MKA1.Types.Components;
using System.Collections.Generic;
var sdk = new SDK(bearerAuth: "<YOUR_BEARER_TOKEN_HERE>");
var res = await sdk.Llm.Evals.CreateRunAsync(body: new MeetKai.MKA1.Types.Components.CreateEvalRunRequest() {
SuiteId = "eval_suite_aa87e2b1112a455b8deabed784372198",
Models = new List<string>() {
"auto",
},
JudgeModel = "auto",
EmbeddingModel = "auto",
Generation = new EvalGenerationRequestConfig() {
Temperature = 0D,
MaxGenToks = 512,
Until = new List<string>() {
"<|endoftext|>",
},
DoSample = false,
ChatTemplateKwargs = new Dictionary<string, object>() {
{ "enable_thinking", false },
},
TimeoutSeconds = 120,
MaxRetries = 2,
MaxEmptyRetries = 1,
},
GenerationConcurrency = 4,
GraderConcurrency = 2,
MaxWorkflowSampleActivities = 5000,
Metadata = new Dictionary<string, string>() {
{ "purpose", "mvp" },
},
});
// handle responsecurl --request POST \
--url https://apigw.mka1.com/api/v1/llm/evals/runs \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"suite_id": "eval_suite_aa87e2b1112a455b8deabed784372198",
"models": [
"auto"
],
"judge_model": "auto",
"embedding_model": "auto",
"generation": {
"temperature": 0,
"max_gen_toks": 512,
"until": [
"<|endoftext|>"
],
"do_sample": false,
"chat_template_kwargs": {
"enable_thinking": false
},
"max_retries": 2,
"max_empty_retries": 1,
"timeout_seconds": 120
},
"generation_concurrency": 4,
"grader_concurrency": 2,
"max_workflow_sample_activities": 5000,
"metadata": {
"purpose": "mvp"
}
}
'const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
suite_id: 'eval_suite_aa87e2b1112a455b8deabed784372198',
models: ['auto'],
judge_model: 'auto',
embedding_model: 'auto',
generation: {
temperature: 0,
max_gen_toks: 512,
until: ['<|endoftext|>'],
do_sample: false,
chat_template_kwargs: {enable_thinking: false},
max_retries: 2,
max_empty_retries: 1,
timeout_seconds: 120
},
generation_concurrency: 4,
grader_concurrency: 2,
max_workflow_sample_activities: 5000,
metadata: {purpose: 'mvp'}
})
};
fetch('https://apigw.mka1.com/api/v1/llm/evals/runs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://apigw.mka1.com/api/v1/llm/evals/runs",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'suite_id' => 'eval_suite_aa87e2b1112a455b8deabed784372198',
'models' => [
'auto'
],
'judge_model' => 'auto',
'embedding_model' => 'auto',
'generation' => [
'temperature' => 0,
'max_gen_toks' => 512,
'until' => [
'<|endoftext|>'
],
'do_sample' => false,
'chat_template_kwargs' => [
'enable_thinking' => false
],
'max_retries' => 2,
'max_empty_retries' => 1,
'timeout_seconds' => 120
],
'generation_concurrency' => 4,
'grader_concurrency' => 2,
'max_workflow_sample_activities' => 5000,
'metadata' => [
'purpose' => 'mvp'
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://apigw.mka1.com/api/v1/llm/evals/runs"
payload := strings.NewReader("{\n \"suite_id\": \"eval_suite_aa87e2b1112a455b8deabed784372198\",\n \"models\": [\n \"auto\"\n ],\n \"judge_model\": \"auto\",\n \"embedding_model\": \"auto\",\n \"generation\": {\n \"temperature\": 0,\n \"max_gen_toks\": 512,\n \"until\": [\n \"<|endoftext|>\"\n ],\n \"do_sample\": false,\n \"chat_template_kwargs\": {\n \"enable_thinking\": false\n },\n \"max_retries\": 2,\n \"max_empty_retries\": 1,\n \"timeout_seconds\": 120\n },\n \"generation_concurrency\": 4,\n \"grader_concurrency\": 2,\n \"max_workflow_sample_activities\": 5000,\n \"metadata\": {\n \"purpose\": \"mvp\"\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://apigw.mka1.com/api/v1/llm/evals/runs")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"suite_id\": \"eval_suite_aa87e2b1112a455b8deabed784372198\",\n \"models\": [\n \"auto\"\n ],\n \"judge_model\": \"auto\",\n \"embedding_model\": \"auto\",\n \"generation\": {\n \"temperature\": 0,\n \"max_gen_toks\": 512,\n \"until\": [\n \"<|endoftext|>\"\n ],\n \"do_sample\": false,\n \"chat_template_kwargs\": {\n \"enable_thinking\": false\n },\n \"max_retries\": 2,\n \"max_empty_retries\": 1,\n \"timeout_seconds\": 120\n },\n \"generation_concurrency\": 4,\n \"grader_concurrency\": 2,\n \"max_workflow_sample_activities\": 5000,\n \"metadata\": {\n \"purpose\": \"mvp\"\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://apigw.mka1.com/api/v1/llm/evals/runs")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"suite_id\": \"eval_suite_aa87e2b1112a455b8deabed784372198\",\n \"models\": [\n \"auto\"\n ],\n \"judge_model\": \"auto\",\n \"embedding_model\": \"auto\",\n \"generation\": {\n \"temperature\": 0,\n \"max_gen_toks\": 512,\n \"until\": [\n \"<|endoftext|>\"\n ],\n \"do_sample\": false,\n \"chat_template_kwargs\": {\n \"enable_thinking\": false\n },\n \"max_retries\": 2,\n \"max_empty_retries\": 1,\n \"timeout_seconds\": 120\n },\n \"generation_concurrency\": 4,\n \"grader_concurrency\": 2,\n \"max_workflow_sample_activities\": 5000,\n \"metadata\": {\n \"purpose\": \"mvp\"\n }\n}"
response = http.request(request)
puts response.read_body{
"id": "eval_run_aa87e2b1112a455b8deabed784372198",
"object": "eval.run",
"suite_id": "eval_suite_aa87e2b1112a455b8deabed784372198",
"suite_version": 1,
"suite_version_id": "eval_sver_aa87e2b1112a455b8deabed784372198",
"org_id": "org-acme",
"team_id": "team-research",
"status": "in_progress",
"models": [
"auto"
],
"task_ids": null,
"judge_model": "auto",
"embedding_model": "auto",
"generation": {
"temperature": 0,
"max_output_tokens": 512
},
"request_counts": {
"total": 100,
"completed": 10,
"failed": 0
},
"metrics": null,
"error": null,
"artifact_file_ids": [],
"metadata": {
"purpose": "mvp"
},
"agent_name": null,
"agent_version": null,
"agent_effort": null,
"cost_usd": null,
"created_at": 1704067200,
"started_at": 1704067210,
"completed_at": null,
"cancelled_at": null,
"failed_at": null
}Authorizations
Gateway auth: send Authorization: Bearer <mka1-api-key>. For multi-user server-side integrations, you can also send X-On-Behalf-Of: <external-user-id>.
Headers
Optional external end-user identifier forwarded by the API gateway.
Body
application/json
Minimum string length:
1Required array length:
1 - 20 elementsRequired string length:
1 - 255Required range:
1 <= x <= 2147483647Minimum array length:
1Show child attributes
Show child attributes
Required range:
1 <= x <= 256Required range:
1 <= x <= 256Required range:
1 <= x <= 256Required range:
1 <= x <= 9007199254740991Maximum sample-stage activity reservations per Temporal workflow execution before continuing as new.
Required range:
100 <= x <= 50000Show child attributes
Show child attributes
Response
200 - application/json
OK
Required range:
-9007199254740991 <= x <= 9007199254740991The org that owns this run.
The team that owns this run.
Available options:
queued, in_progress, finalizing, completed, failed, cancelling, cancelled Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Coding-agent harness that produced this run (e.g. omp, claude-code). Null for every non-harness eval kind — a null here is not a missing value, it means the run is not an agent run.
Version of agent_name, as the harness reported it. Null when agent_name is.
Reasoning-effort setting the agent ran at (e.g. medium, xhigh). Part of the leaderboard's row identity: the same agent and model at two efforts are two rows, not one. Null when agent_name is.
Total spend for the run in USD. Null when the harness did not report cost.
Token totals for the run, summed over its samples when it finalized. Each is null when nothing reported that figure, which is a different fact from zero and must not render as free.
Show child attributes
Show child attributes
Required range:
-9007199254740991 <= x <= 9007199254740991Required range:
-9007199254740991 <= x <= 9007199254740991Required range:
-9007199254740991 <= x <= 9007199254740991Required range:
-9007199254740991 <= x <= 9007199254740991Required range:
-9007199254740991 <= x <= 9007199254740991Was this page helpful?
⌘I