Evaluation API Reference
评测引擎、指标目录、同步检查、异步 job 和报告接口。
Evaluation 为质量、RAG、Agent、多轮、自定义和性能工作负载返回 scorecard 与指标级结果。下面的 API 示例以 AI 生成的 Python、JavaScript 和 Java 客户端代码展示。
| 方法 | 路径 | Scope | 作用 |
|---|---|---|---|
GET | /v1/eval/engines | eval:read | 列出可用评测引擎。 |
GET | /v1/eval/metrics | eval:read | 列出指标定义和 engine binding。 |
POST | /v1/eval/sync | eval:write | 运行小规模同步评测。 |
POST | /v1/eval/jobs | eval:write | 提交异步评测 job。 |
GET | /v1/eval/jobs/{jobId}/result | eval:read | 读取已完成评测报告。 |
引擎和指标
import osimport requestsBASE_URL = os.getenv("CORTEX_URL", "http://127.0.0.1:8080")TOKEN = os.getenv("CORTEX_TOKEN", "replace_with_token")def auth_headers(): return {"Authorization": f"Bearer {TOKEN}"}for path in ["/v1/eval/engines", "/v1/eval/metrics"]: response = requests.get(f"{BASE_URL}{path}", headers=auth_headers()) response.raise_for_status() print(path, response.json())const BASE_URL = process.env.CORTEX_URL ?? "http://127.0.0.1:8080";const TOKEN = process.env.CORTEX_TOKEN ?? "replace_with_token";const authHeaders = { Authorization: `Bearer ${TOKEN}`,};for (const path of ["/v1/eval/engines", "/v1/eval/metrics"]) { const response = await fetch(`${BASE_URL}${path}`, { headers: authHeaders }); if (!response.ok) throw new Error(await response.text()); console.log(path, await response.json());}import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;public class CortexExample { static final String BASE_URL = System.getenv().getOrDefault("CORTEX_URL", "http://127.0.0.1:8080"); static final String TOKEN = System.getenv().getOrDefault("CORTEX_TOKEN", "replace_with_token"); static final HttpClient HTTP = HttpClient.newHttpClient(); static void print(HttpResponse<String> response) { System.out.println(response.statusCode()); System.out.println(response.body()); } public static void main(String[] args) throws Exception { for (String path : new String[] {"/v1/eval/engines", "/v1/eval/metrics"}) { HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + path)) .header("Authorization", "Bearer " + TOKEN) .GET() .build(); print(HTTP.send(request, HttpResponse.BodyHandlers.ofString())); } }}RAG 同步请求
import osimport requestsBASE_URL = os.getenv("CORTEX_URL", "http://127.0.0.1:8080")TOKEN = os.getenv("CORTEX_TOKEN", "replace_with_token")def auth_headers(): return {"Authorization": f"Bearer {TOKEN}"}payload = { "name": "quickstart-rag-eval", "eval_type": "rag", "engine_id": "deepeval", "input": { "type": "inline_test_cases", "test_cases": [ { "user_input": "What does Cortex Parse do?", "actual_output": "Cortex Parse converts URLs and files into normalized Markdown.", "expected_output": "Parse should mention URLs, files, and Markdown.", "retrieval_contexts": [ "Cortex Parse accepts URLs and files and returns normalized Markdown with metadata." ], "metadata": { "case_id": "rag-001" } } ] }, "target": { "type": "existing_outputs" }, "metrics": [ { "metric_key": "rag.answer_relevance", "threshold": 0.65 }, { "metric_key": "rag.faithfulness", "threshold": 0.65 }, { "metric_key": "rag.contextual_relevance", "threshold": 0.6 } ], "output": { "persist_report_object": True, "include_sample_results": True }}response = requests.post( f"{BASE_URL}/v1/eval/sync", headers={**auth_headers(), "Content-Type": "application/json"}, json=payload,)response.raise_for_status()data = response.json()print(data)const BASE_URL = process.env.CORTEX_URL ?? "http://127.0.0.1:8080";const TOKEN = process.env.CORTEX_TOKEN ?? "replace_with_token";const authHeaders = { Authorization: `Bearer ${TOKEN}`,};const payload = { "name": "quickstart-rag-eval", "eval_type": "rag", "engine_id": "deepeval", "input": { "type": "inline_test_cases", "test_cases": [ { "user_input": "What does Cortex Parse do?", "actual_output": "Cortex Parse converts URLs and files into normalized Markdown.", "expected_output": "Parse should mention URLs, files, and Markdown.", "retrieval_contexts": [ "Cortex Parse accepts URLs and files and returns normalized Markdown with metadata." ], "metadata": { "case_id": "rag-001" } } ] }, "target": { "type": "existing_outputs" }, "metrics": [ { "metric_key": "rag.answer_relevance", "threshold": 0.65 }, { "metric_key": "rag.faithfulness", "threshold": 0.65 }, { "metric_key": "rag.contextual_relevance", "threshold": 0.6 } ], "output": { "persist_report_object": true, "include_sample_results": true }};const response = await fetch(`${BASE_URL}/v1/eval/sync`, { method: "POST", headers: { ...authHeaders, "Content-Type": "application/json" }, body: JSON.stringify(payload),});if (!response.ok) throw new Error(await response.text());const data = await response.json();console.log(data);import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;public class CortexExample { static final String BASE_URL = System.getenv().getOrDefault("CORTEX_URL", "http://127.0.0.1:8080"); static final String TOKEN = System.getenv().getOrDefault("CORTEX_TOKEN", "replace_with_token"); static final HttpClient HTTP = HttpClient.newHttpClient(); static void print(HttpResponse<String> response) { System.out.println(response.statusCode()); System.out.println(response.body()); } public static void main(String[] args) throws Exception { String json = """ { \"name\": \"quickstart-rag-eval\", \"eval_type\": \"rag\", \"engine_id\": \"deepeval\", \"input\": { \"type\": \"inline_test_cases\", \"test_cases\": [ { \"user_input\": \"What does Cortex Parse do?\", \"actual_output\": \"Cortex Parse converts URLs and files into normalized Markdown.\", \"expected_output\": \"Parse should mention URLs, files, and Markdown.\", \"retrieval_contexts\": [ \"Cortex Parse accepts URLs and files and returns normalized Markdown with metadata.\" ], \"metadata\": { \"case_id\": \"rag-001\" } } ] }, \"target\": { \"type\": \"existing_outputs\" }, \"metrics\": [ { \"metric_key\": \"rag.answer_relevance\", \"threshold\": 0.65 }, { \"metric_key\": \"rag.faithfulness\", \"threshold\": 0.65 }, { \"metric_key\": \"rag.contextual_relevance\", \"threshold\": 0.6 } ], \"output\": { \"persist_report_object\": true, \"include_sample_results\": true } } """; HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + "/v1/eval/sync")) .header("Authorization", "Bearer " + TOKEN) .header("Content-Type", "application/json") .POST(HttpRequest.BodyPublishers.ofString(json)) .build(); print(HTTP.send(request, HttpResponse.BodyHandlers.ofString())); }}异步评测 job
import osimport requestsBASE_URL = os.getenv("CORTEX_URL", "http://127.0.0.1:8080")TOKEN = os.getenv("CORTEX_TOKEN", "replace_with_token")def auth_headers(): return {"Authorization": f"Bearer {TOKEN}"}payload = { "name": "quickstart-rag-eval", "eval_type": "rag", "engine_id": "deepeval", "input": { "type": "inline_test_cases", "test_cases": [ { "user_input": "What does Cortex Parse do?", "actual_output": "Cortex Parse converts URLs and files into normalized Markdown.", "expected_output": "Parse should mention URLs, files, and Markdown.", "retrieval_contexts": [ "Cortex Parse accepts URLs and files and returns normalized Markdown with metadata." ], "metadata": { "case_id": "rag-001" } } ] }, "target": { "type": "existing_outputs" }, "metrics": [ { "metric_key": "rag.answer_relevance", "threshold": 0.65 }, { "metric_key": "rag.faithfulness", "threshold": 0.65 }, { "metric_key": "rag.contextual_relevance", "threshold": 0.6 } ], "output": { "persist_report_object": True, "include_sample_results": True }}response = requests.post( f"{BASE_URL}/v1/eval/jobs", headers={**auth_headers(), "Content-Type": "application/json"}, json=payload,)response.raise_for_status()data = response.json()print(data)const BASE_URL = process.env.CORTEX_URL ?? "http://127.0.0.1:8080";const TOKEN = process.env.CORTEX_TOKEN ?? "replace_with_token";const authHeaders = { Authorization: `Bearer ${TOKEN}`,};const payload = { "name": "quickstart-rag-eval", "eval_type": "rag", "engine_id": "deepeval", "input": { "type": "inline_test_cases", "test_cases": [ { "user_input": "What does Cortex Parse do?", "actual_output": "Cortex Parse converts URLs and files into normalized Markdown.", "expected_output": "Parse should mention URLs, files, and Markdown.", "retrieval_contexts": [ "Cortex Parse accepts URLs and files and returns normalized Markdown with metadata." ], "metadata": { "case_id": "rag-001" } } ] }, "target": { "type": "existing_outputs" }, "metrics": [ { "metric_key": "rag.answer_relevance", "threshold": 0.65 }, { "metric_key": "rag.faithfulness", "threshold": 0.65 }, { "metric_key": "rag.contextual_relevance", "threshold": 0.6 } ], "output": { "persist_report_object": true, "include_sample_results": true }};const response = await fetch(`${BASE_URL}/v1/eval/jobs`, { method: "POST", headers: { ...authHeaders, "Content-Type": "application/json" }, body: JSON.stringify(payload),});if (!response.ok) throw new Error(await response.text());const data = await response.json();console.log(data);import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;public class CortexExample { static final String BASE_URL = System.getenv().getOrDefault("CORTEX_URL", "http://127.0.0.1:8080"); static final String TOKEN = System.getenv().getOrDefault("CORTEX_TOKEN", "replace_with_token"); static final HttpClient HTTP = HttpClient.newHttpClient(); static void print(HttpResponse<String> response) { System.out.println(response.statusCode()); System.out.println(response.body()); } public static void main(String[] args) throws Exception { String json = """ { \"name\": \"quickstart-rag-eval\", \"eval_type\": \"rag\", \"engine_id\": \"deepeval\", \"input\": { \"type\": \"inline_test_cases\", \"test_cases\": [ { \"user_input\": \"What does Cortex Parse do?\", \"actual_output\": \"Cortex Parse converts URLs and files into normalized Markdown.\", \"expected_output\": \"Parse should mention URLs, files, and Markdown.\", \"retrieval_contexts\": [ \"Cortex Parse accepts URLs and files and returns normalized Markdown with metadata.\" ], \"metadata\": { \"case_id\": \"rag-001\" } } ] }, \"target\": { \"type\": \"existing_outputs\" }, \"metrics\": [ { \"metric_key\": \"rag.answer_relevance\", \"threshold\": 0.65 }, { \"metric_key\": \"rag.faithfulness\", \"threshold\": 0.65 }, { \"metric_key\": \"rag.contextual_relevance\", \"threshold\": 0.6 } ], \"output\": { \"persist_report_object\": true, \"include_sample_results\": true } } """; HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + "/v1/eval/jobs")) .header("Authorization", "Bearer " + TOKEN) .header("Content-Type", "application/json") .POST(HttpRequest.BodyPublishers.ofString(json)) .build(); print(HTTP.send(request, HttpResponse.BodyHandlers.ofString())); }}EvalScope 性能请求
import osimport requestsBASE_URL = os.getenv("CORTEX_URL", "http://127.0.0.1:8080")TOKEN = os.getenv("CORTEX_TOKEN", "replace_with_token")def auth_headers(): return {"Authorization": f"Bearer {TOKEN}"}payload = { "name": "perf-smoke", "eval_type": "perf", "engine_id": "evalscope", "input": { "type": "builtin_dataset", "builtin_dataset_key": "openqa" }, "target": { "type": "api", "protocol": "openai_compatible", "endpoint_url": "http://127.0.0.1:8080/v1/health/ready", "timeout_seconds": 60 }, "metrics": [ { "metric_key": "perf.qps", "threshold": 1 }, { "metric_key": "perf.p99_latency", "threshold": 5 } ], "engine_options": { "parallel": [ 1, 2 ], "number": [ 10 ], "stream": False }}response = requests.post( f"{BASE_URL}/v1/eval/jobs", headers={**auth_headers(), "Content-Type": "application/json"}, json=payload,)response.raise_for_status()data = response.json()print(data)const BASE_URL = process.env.CORTEX_URL ?? "http://127.0.0.1:8080";const TOKEN = process.env.CORTEX_TOKEN ?? "replace_with_token";const authHeaders = { Authorization: `Bearer ${TOKEN}`,};const payload = { "name": "perf-smoke", "eval_type": "perf", "engine_id": "evalscope", "input": { "type": "builtin_dataset", "builtin_dataset_key": "openqa" }, "target": { "type": "api", "protocol": "openai_compatible", "endpoint_url": "http://127.0.0.1:8080/v1/health/ready", "timeout_seconds": 60 }, "metrics": [ { "metric_key": "perf.qps", "threshold": 1 }, { "metric_key": "perf.p99_latency", "threshold": 5 } ], "engine_options": { "parallel": [ 1, 2 ], "number": [ 10 ], "stream": false }};const response = await fetch(`${BASE_URL}/v1/eval/jobs`, { method: "POST", headers: { ...authHeaders, "Content-Type": "application/json" }, body: JSON.stringify(payload),});if (!response.ok) throw new Error(await response.text());const data = await response.json();console.log(data);import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;public class CortexExample { static final String BASE_URL = System.getenv().getOrDefault("CORTEX_URL", "http://127.0.0.1:8080"); static final String TOKEN = System.getenv().getOrDefault("CORTEX_TOKEN", "replace_with_token"); static final HttpClient HTTP = HttpClient.newHttpClient(); static void print(HttpResponse<String> response) { System.out.println(response.statusCode()); System.out.println(response.body()); } public static void main(String[] args) throws Exception { String json = """ { \"name\": \"perf-smoke\", \"eval_type\": \"perf\", \"engine_id\": \"evalscope\", \"input\": { \"type\": \"builtin_dataset\", \"builtin_dataset_key\": \"openqa\" }, \"target\": { \"type\": \"api\", \"protocol\": \"openai_compatible\", \"endpoint_url\": \"http://127.0.0.1:8080/v1/health/ready\", \"timeout_seconds\": 60 }, \"metrics\": [ { \"metric_key\": \"perf.qps\", \"threshold\": 1 }, { \"metric_key\": \"perf.p99_latency\", \"threshold\": 5 } ], \"engine_options\": { \"parallel\": [ 1, 2 ], \"number\": [ 10 ], \"stream\": false } } """; HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + "/v1/eval/jobs")) .header("Authorization", "Bearer " + TOKEN) .header("Content-Type", "application/json") .POST(HttpRequest.BodyPublishers.ofString(json)) .build(); print(HTTP.send(request, HttpResponse.BodyHandlers.ofString())); }}