Skip to main content
POST
/
api
/
v1
/
llm
/
evals
/
runs
Python (SDK)
from meetkai_mka1 import SDK, models


with SDK(
    bearer_auth="<YOUR_BEARER_TOKEN_HERE>",
) as sdk:

    res = sdk.llm.evals.create_run(suite_id="eval_suite_aa87e2b1112a455b8deabed784372198", models=[
        "auto",
    ], judge_model="auto", embedding_model="auto", generation=models.EvalGenerationConfig(
        temperature=0,
        max_gen_toks=512,
        until=[
            "<|endoftext|>",
        ],
        do_sample=False,
        chat_template_kwargs={
            "enable_thinking": False,
        },
        timeout_seconds=120,
        max_retries=2,
        max_empty_retries=1,
    ), generation_concurrency=4, grader_concurrency=2, max_workflow_sample_activities=5000, metadata={
        "purpose": "mvp",
    })

    # Handle response
    print(res)
import { SDK } from "@meetkai/mka1";

const sdk = new SDK({
  bearerAuth: "<YOUR_BEARER_TOKEN_HERE>",
});

async function run() {
  const result = await sdk.llm.evals.createRun({
    createEvalRunRequest: {
      suiteId: "eval_suite_aa87e2b1112a455b8deabed784372198",
      models: [
        "auto",
      ],
      judgeModel: "auto",
      embeddingModel: "auto",
      generation: {
        temperature: 0,
        maxGenToks: 512,
        until: [
          "<|endoftext|>",
        ],
        doSample: false,
        chatTemplateKwargs: {
          "enable_thinking": false,
        },
        timeoutSeconds: 120,
        maxRetries: 2,
        maxEmptyRetries: 1,
      },
      generationConcurrency: 4,
      graderConcurrency: 2,
      maxWorkflowSampleActivities: 5000,
      metadata: {
        "purpose": "mvp",
      },
    },
  });

  console.log(result);
}

run();
using MeetKai.MKA1;
using MeetKai.MKA1.Types.Components;
using System.Collections.Generic;

var sdk = new SDK(bearerAuth: "<YOUR_BEARER_TOKEN_HERE>");

var res = await sdk.Llm.Evals.CreateRunAsync(body: new MeetKai.MKA1.Types.Components.CreateEvalRunRequest() {
    SuiteId = "eval_suite_aa87e2b1112a455b8deabed784372198",
    Models = new List<string>() {
        "auto",
    },
    JudgeModel = "auto",
    EmbeddingModel = "auto",
    Generation = new EvalGenerationConfig() {
        Temperature = 0D,
        MaxGenToks = 512,
        Until = new List<string>() {
            "<|endoftext|>",
        },
        DoSample = false,
        ChatTemplateKwargs = new Dictionary<string, object>() {
            { "enable_thinking", false },
        },
        TimeoutSeconds = 120,
        MaxRetries = 2,
        MaxEmptyRetries = 1,
    },
    GenerationConcurrency = 4,
    GraderConcurrency = 2,
    MaxWorkflowSampleActivities = 5000,
    Metadata = new Dictionary<string, string>() {
        { "purpose", "mvp" },
    },
});

// handle response
curl --request POST \
  --url https://apigw.mka1.com/api/v1/llm/evals/runs \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "suite_id": "eval_suite_aa87e2b1112a455b8deabed784372198",
  "models": [
    "auto"
  ],
  "judge_model": "auto",
  "embedding_model": "auto",
  "generation": {
    "temperature": 0,
    "max_gen_toks": 512,
    "until": [
      "<|endoftext|>"
    ],
    "do_sample": false,
    "chat_template_kwargs": {
      "enable_thinking": false
    },
    "max_retries": 2,
    "max_empty_retries": 1,
    "timeout_seconds": 120
  },
  "generation_concurrency": 4,
  "grader_concurrency": 2,
  "max_workflow_sample_activities": 5000,
  "metadata": {
    "purpose": "mvp"
  }
}
'
const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    suite_id: 'eval_suite_aa87e2b1112a455b8deabed784372198',
    models: ['auto'],
    judge_model: 'auto',
    embedding_model: 'auto',
    generation: {
      temperature: 0,
      max_gen_toks: 512,
      until: ['<|endoftext|>'],
      do_sample: false,
      chat_template_kwargs: {enable_thinking: false},
      max_retries: 2,
      max_empty_retries: 1,
      timeout_seconds: 120
    },
    generation_concurrency: 4,
    grader_concurrency: 2,
    max_workflow_sample_activities: 5000,
    metadata: {purpose: 'mvp'}
  })
};

fetch('https://apigw.mka1.com/api/v1/llm/evals/runs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));
<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://apigw.mka1.com/api/v1/llm/evals/runs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'suite_id' => 'eval_suite_aa87e2b1112a455b8deabed784372198',
    'models' => [
        'auto'
    ],
    'judge_model' => 'auto',
    'embedding_model' => 'auto',
    'generation' => [
        'temperature' => 0,
        'max_gen_toks' => 512,
        'until' => [
                '<|endoftext|>'
        ],
        'do_sample' => false,
        'chat_template_kwargs' => [
                'enable_thinking' => false
        ],
        'max_retries' => 2,
        'max_empty_retries' => 1,
        'timeout_seconds' => 120
    ],
    'generation_concurrency' => 4,
    'grader_concurrency' => 2,
    'max_workflow_sample_activities' => 5000,
    'metadata' => [
        'purpose' => 'mvp'
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}
package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://apigw.mka1.com/api/v1/llm/evals/runs"

	payload := strings.NewReader("{\n  \"suite_id\": \"eval_suite_aa87e2b1112a455b8deabed784372198\",\n  \"models\": [\n    \"auto\"\n  ],\n  \"judge_model\": \"auto\",\n  \"embedding_model\": \"auto\",\n  \"generation\": {\n    \"temperature\": 0,\n    \"max_gen_toks\": 512,\n    \"until\": [\n      \"<|endoftext|>\"\n    ],\n    \"do_sample\": false,\n    \"chat_template_kwargs\": {\n      \"enable_thinking\": false\n    },\n    \"max_retries\": 2,\n    \"max_empty_retries\": 1,\n    \"timeout_seconds\": 120\n  },\n  \"generation_concurrency\": 4,\n  \"grader_concurrency\": 2,\n  \"max_workflow_sample_activities\": 5000,\n  \"metadata\": {\n    \"purpose\": \"mvp\"\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}
HttpResponse<String> response = Unirest.post("https://apigw.mka1.com/api/v1/llm/evals/runs")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"suite_id\": \"eval_suite_aa87e2b1112a455b8deabed784372198\",\n  \"models\": [\n    \"auto\"\n  ],\n  \"judge_model\": \"auto\",\n  \"embedding_model\": \"auto\",\n  \"generation\": {\n    \"temperature\": 0,\n    \"max_gen_toks\": 512,\n    \"until\": [\n      \"<|endoftext|>\"\n    ],\n    \"do_sample\": false,\n    \"chat_template_kwargs\": {\n      \"enable_thinking\": false\n    },\n    \"max_retries\": 2,\n    \"max_empty_retries\": 1,\n    \"timeout_seconds\": 120\n  },\n  \"generation_concurrency\": 4,\n  \"grader_concurrency\": 2,\n  \"max_workflow_sample_activities\": 5000,\n  \"metadata\": {\n    \"purpose\": \"mvp\"\n  }\n}")
  .asString();
require 'uri'
require 'net/http'

url = URI("https://apigw.mka1.com/api/v1/llm/evals/runs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"suite_id\": \"eval_suite_aa87e2b1112a455b8deabed784372198\",\n  \"models\": [\n    \"auto\"\n  ],\n  \"judge_model\": \"auto\",\n  \"embedding_model\": \"auto\",\n  \"generation\": {\n    \"temperature\": 0,\n    \"max_gen_toks\": 512,\n    \"until\": [\n      \"<|endoftext|>\"\n    ],\n    \"do_sample\": false,\n    \"chat_template_kwargs\": {\n      \"enable_thinking\": false\n    },\n    \"max_retries\": 2,\n    \"max_empty_retries\": 1,\n    \"timeout_seconds\": 120\n  },\n  \"generation_concurrency\": 4,\n  \"grader_concurrency\": 2,\n  \"max_workflow_sample_activities\": 5000,\n  \"metadata\": {\n    \"purpose\": \"mvp\"\n  }\n}"

response = http.request(request)
puts response.read_body
{
  "id": "eval_run_aa87e2b1112a455b8deabed784372198",
  "object": "eval.run",
  "suite_id": "eval_suite_aa87e2b1112a455b8deabed784372198",
  "suite_version": 1,
  "suite_version_id": "eval_sver_aa87e2b1112a455b8deabed784372198",
  "status": "in_progress",
  "models": [
    "auto"
  ],
  "task_ids": null,
  "judge_model": "auto",
  "embedding_model": "auto",
  "generation": {
    "temperature": 0,
    "max_output_tokens": 512
  },
  "request_counts": {
    "total": 100,
    "completed": 10,
    "failed": 0
  },
  "metrics": null,
  "error": null,
  "artifact_file_ids": [],
  "metadata": {
    "purpose": "mvp"
  },
  "created_at": 1704067200,
  "started_at": 1704067210,
  "completed_at": null,
  "cancelled_at": null,
  "failed_at": null
}

Authorizations

​
Authorization
string
header
required

Gateway auth: send Authorization: Bearer <mka1-api-key>. For multi-user server-side integrations, you can also send X-On-Behalf-Of: <external-user-id>.

Headers

​
X-On-Behalf-Of
string

Optional external end-user identifier forwarded by the API gateway.

Body

application/json
​
suite_id
string
required
​
models
string[]
required
Required array length: 1 - 20 elements
Minimum string length: 1
​
suite_version
integer
Required range: 1 <= x <= 9007199254740991
​
task_ids
string[]
Minimum array length: 1
​
judge_model
string
​
embedding_model
string
​
generation
object
​
generation_concurrency
integer
Required range: 1 <= x <= 256
​
concurrency
integer
Required range: 1 <= x <= 256
​
grader_concurrency
integer
Required range: 1 <= x <= 256
​
max_samples_per_task
integer
Required range: 1 <= x <= 9007199254740991
​
max_workflow_sample_activities
integer

Maximum sample-stage activity reservations per Temporal workflow execution before continuing as new.

Required range: 100 <= x <= 50000
​
metadata
object

Response

200 - application/json

OK

​
id
string
required
​
object
any
required
​
suite_id
string
required
​
suite_version
integer
required
Required range: -9007199254740991 <= x <= 9007199254740991
​
suite_version_id
string
required
​
status
enum<string>
required
Available options:
queued,
in_progress,
finalizing,
completed,
failed,
cancelling,
cancelled
​
models
string[]
required
​
task_ids
string[] | null
required
​
judge_model
string | null
required
​
embedding_model
string | null
required
​
generation
object
required
​
request_counts
object
required
​
metrics
object | null
required
​
error
object | null
required
​
artifact_file_ids
string[]
required
​
metadata
object
required
​
created_at
integer
required
Required range: -9007199254740991 <= x <= 9007199254740991
​
started_at
integer | null
required
Required range: -9007199254740991 <= x <= 9007199254740991
​
completed_at
integer | null
required
Required range: -9007199254740991 <= x <= 9007199254740991
​
cancelled_at
integer | null
required
Required range: -9007199254740991 <= x <= 9007199254740991
​
failed_at
integer | null
required
Required range: -9007199254740991 <= x <= 9007199254740991