from meetkai_mka1 import SDK


with SDK(
    bearer_auth="<YOUR_BEARER_TOKEN_HERE>",
) as sdk:

    res = sdk.llm.evals.create_suite(name="Multilingual QA smoke suite", manifest={
        "tasks": [
            {
                "id": "spanish_qa",
                "type": "custom",
                "dataset": {
                    "source": "huggingface",
                    "path": "IIC/AQuAS",
                    "split": "test",
                },
                "prompt_template": "Responde usando el contexto.\n\nContexto: {{context}}\n\nPregunta: {{question}}\n\nRespuesta:",
                "target_template": "{{answer}}",
                "grader": {
                    "type": "python",
                    "contract": "model_backed",
                    "model_access": "mka1",
                    "file_id": "file_grader123",
                },
                "preprocess": {
                    "type": "python",
                    "source": "def transform(row):\n    return row\n",
                },
                "num_fewshot": 1,
            },
        ],
    }, description="Declarative eval tasks backed by uploaded files or Hugging Face datasets and Python graders.", metadata={
        "owner": "eval-team",
    })

    # Handle response
    print(res)

{
  "id": "eval_suite_aa87e2b1112a455b8deabed784372198",
  "object": "eval.suite",
  "name": "Multilingual QA smoke suite",
  "description": "Declarative eval tasks backed by uploaded files or Hugging Face datasets and Python graders.",
  "active_version": 1,
  "latest_version": 1,
  "metadata": {
    "owner": "eval-team"
  },
  "created_at": 1704067200,
  "updated_at": 1704067200
}