Evaluation 和 Synthesis
通过 Cortex Evaluation 和 Synthesis Job 运行评测、生成数据，并持久化报告与输出对象。
Cortex Evaluation 和 Synthesis 复用 Parse 和 Knowledge 使用的 Job、Storage、授权和遥测基础能力。下面的 API 片段是 AI 生成的 Python、JavaScript 和 Java 客户端示例。
列出评测引擎和指标
import osimport requestsBASE_URL = os.getenv("CORTEX_URL", "http://127.0.0.1:8080")TOKEN = os.getenv("CORTEX_TOKEN", "replace_with_token")def auth_headers(): return {"Authorization": f"Bearer {TOKEN}"}for path in ["/v1/eval/engines", "/v1/eval/metrics"]: response = requests.get(f"{BASE_URL}{path}", headers=auth_headers()) response.raise_for_status() print(path, response.json())const BASE_URL = process.env.CORTEX_URL ?? "http://127.0.0.1:8080";const TOKEN = process.env.CORTEX_TOKEN ?? "replace_with_token";const authHeaders = { Authorization: `Bearer ${TOKEN}`,};for (const path of ["/v1/eval/engines", "/v1/eval/metrics"]) { const response = await fetch(`${BASE_URL}${path}`, { headers: authHeaders }); if (!response.ok) throw new Error(await response.text()); console.log(path, await response.json());}import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;public class CortexExample { static final String BASE_URL = System.getenv().getOrDefault("CORTEX_URL", "http://127.0.0.1:8080"); static final String TOKEN = System.getenv().getOrDefault("CORTEX_TOKEN", "replace_with_token"); static final HttpClient HTTP = HttpClient.newHttpClient(); static void print(HttpResponse<String> response) { System.out.println(response.statusCode()); System.out.println(response.body()); } public static void main(String[] args) throws Exception { for (String path : new String[] {"/v1/eval/engines", "/v1/eval/metrics"}) { HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + path)) .header("Authorization", "Bearer " + TOKEN) .GET() .build(); print(HTTP.send(request, HttpResponse.BodyHandlers.ofString())); } }}提交 RAG 评测 Job
import osimport requestsBASE_URL = os.getenv("CORTEX_URL", "http://127.0.0.1:8080")TOKEN = os.getenv("CORTEX_TOKEN", "replace_with_token")def auth_headers(): return {"Authorization": f"Bearer {TOKEN}"}payload = { "name": "quickstart-rag-eval", "eval_type": "rag", "engine_id": "deepeval", "input": { "type": "inline_test_cases", "test_cases": [ { "user_input": "What does Cortex Parse do?", "actual_output": "Cortex Parse converts URLs and files into normalized Markdown.", "expected_output": "Parse should mention URLs, files, and Markdown.", "retrieval_contexts": [ "Cortex Parse accepts URLs and files and returns normalized Markdown with metadata." ], "metadata": { "case_id": "rag-001" } } ] }, "target": { "type": "existing_outputs" }, "metrics": [ { "metric_key": "rag.answer_relevance", "threshold": 0.65 }, { "metric_key": "rag.faithfulness", "threshold": 0.65 }, { "metric_key": "rag.contextual_relevance", "threshold": 0.6 } ], "output": { "persist_report_object": True, "include_sample_results": True }}response = requests.post( f"{BASE_URL}/v1/eval/jobs", headers={**auth_headers(), "Content-Type": "application/json"}, json=payload,)response.raise_for_status()data = response.json()print(data)const BASE_URL = process.env.CORTEX_URL ?? "http://127.0.0.1:8080";const TOKEN = process.env.CORTEX_TOKEN ?? "replace_with_token";const authHeaders = { Authorization: `Bearer ${TOKEN}`,};const payload = { "name": "quickstart-rag-eval", "eval_type": "rag", "engine_id": "deepeval", "input": { "type": "inline_test_cases", "test_cases": [ { "user_input": "What does Cortex Parse do?", "actual_output": "Cortex Parse converts URLs and files into normalized Markdown.", "expected_output": "Parse should mention URLs, files, and Markdown.", "retrieval_contexts": [ "Cortex Parse accepts URLs and files and returns normalized Markdown with metadata." ], "metadata": { "case_id": "rag-001" } } ] }, "target": { "type": "existing_outputs" }, "metrics": [ { "metric_key": "rag.answer_relevance", "threshold": 0.65 }, { "metric_key": "rag.faithfulness", "threshold": 0.65 }, { "metric_key": "rag.contextual_relevance", "threshold": 0.6 } ], "output": { "persist_report_object": true, "include_sample_results": true }};const response = await fetch(`${BASE_URL}/v1/eval/jobs`, { method: "POST", headers: { ...authHeaders, "Content-Type": "application/json" }, body: JSON.stringify(payload),});if (!response.ok) throw new Error(await response.text());const data = await response.json();console.log(data);import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;public class CortexExample { static final String BASE_URL = System.getenv().getOrDefault("CORTEX_URL", "http://127.0.0.1:8080"); static final String TOKEN = System.getenv().getOrDefault("CORTEX_TOKEN", "replace_with_token"); static final HttpClient HTTP = HttpClient.newHttpClient(); static void print(HttpResponse<String> response) { System.out.println(response.statusCode()); System.out.println(response.body()); } public static void main(String[] args) throws Exception { String json = """ { \"name\": \"quickstart-rag-eval\", \"eval_type\": \"rag\", \"engine_id\": \"deepeval\", \"input\": { \"type\": \"inline_test_cases\", \"test_cases\": [ { \"user_input\": \"What does Cortex Parse do?\", \"actual_output\": \"Cortex Parse converts URLs and files into normalized Markdown.\", \"expected_output\": \"Parse should mention URLs, files, and Markdown.\", \"retrieval_contexts\": [ \"Cortex Parse accepts URLs and files and returns normalized Markdown with metadata.\" ], \"metadata\": { \"case_id\": \"rag-001\" } } ] }, \"target\": { \"type\": \"existing_outputs\" }, \"metrics\": [ { \"metric_key\": \"rag.answer_relevance\", \"threshold\": 0.65 }, { \"metric_key\": \"rag.faithfulness\", \"threshold\": 0.65 }, { \"metric_key\": \"rag.contextual_relevance\", \"threshold\": 0.6 } ], \"output\": { \"persist_report_object\": true, \"include_sample_results\": true } } """; HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + "/v1/eval/jobs")) .header("Authorization", "Bearer " + TOKEN) .header("Content-Type", "application/json") .POST(HttpRequest.BodyPublishers.ofString(json)) .build(); print(HTTP.send(request, HttpResponse.BodyHandlers.ofString())); }}轮询并获取结果：
import osimport requestsBASE_URL = os.getenv("CORTEX_URL", "http://127.0.0.1:8080")TOKEN = os.getenv("CORTEX_TOKEN", "replace_with_token")def auth_headers(): return {"Authorization": f"Bearer {TOKEN}"}response = requests.get(f"{BASE_URL}/v1/jobs/job_xxx", headers=auth_headers())response.raise_for_status()data = response.json()print(data)const BASE_URL = process.env.CORTEX_URL ?? "http://127.0.0.1:8080";const TOKEN = process.env.CORTEX_TOKEN ?? "replace_with_token";const authHeaders = { Authorization: `Bearer ${TOKEN}`,};const response = await fetch(`${BASE_URL}/v1/jobs/job_xxx`, { headers: authHeaders,});if (!response.ok) throw new Error(await response.text());const data = await response.json();console.log(data);import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;public class CortexExample { static final String BASE_URL = System.getenv().getOrDefault("CORTEX_URL", "http://127.0.0.1:8080"); static final String TOKEN = System.getenv().getOrDefault("CORTEX_TOKEN", "replace_with_token"); static final HttpClient HTTP = HttpClient.newHttpClient(); static void print(HttpResponse<String> response) { System.out.println(response.statusCode()); System.out.println(response.body()); } public static void main(String[] args) throws Exception { HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + "/v1/jobs/job_xxx")) .header("Authorization", "Bearer " + TOKEN) .GET() .build(); print(HTTP.send(request, HttpResponse.BodyHandlers.ofString())); }}import osimport requestsBASE_URL = os.getenv("CORTEX_URL", "http://127.0.0.1:8080")TOKEN = os.getenv("CORTEX_TOKEN", "replace_with_token")def auth_headers(): return {"Authorization": f"Bearer {TOKEN}"}response = requests.get(f"{BASE_URL}/v1/eval/jobs/job_xxx/result", headers=auth_headers())response.raise_for_status()data = response.json()print(data)const BASE_URL = process.env.CORTEX_URL ?? "http://127.0.0.1:8080";const TOKEN = process.env.CORTEX_TOKEN ?? "replace_with_token";const authHeaders = { Authorization: `Bearer ${TOKEN}`,};const response = await fetch(`${BASE_URL}/v1/eval/jobs/job_xxx/result`, { headers: authHeaders,});if (!response.ok) throw new Error(await response.text());const data = await response.json();console.log(data);import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;public class CortexExample { static final String BASE_URL = System.getenv().getOrDefault("CORTEX_URL", "http://127.0.0.1:8080"); static final String TOKEN = System.getenv().getOrDefault("CORTEX_TOKEN", "replace_with_token"); static final HttpClient HTTP = HttpClient.newHttpClient(); static void print(HttpResponse<String> response) { System.out.println(response.statusCode()); System.out.println(response.body()); } public static void main(String[] args) throws Exception { HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + "/v1/eval/jobs/job_xxx/result")) .header("Authorization", "Bearer " + TOKEN) .GET() .build(); print(HTTP.send(request, HttpResponse.BodyHandlers.ofString())); }}异步评测完成后可以持久化 evaluation_report 对象，并回写 eval_runs.report_object_id。
列出合成引擎
import osimport requestsBASE_URL = os.getenv("CORTEX_URL", "http://127.0.0.1:8080")TOKEN = os.getenv("CORTEX_TOKEN", "replace_with_token")def auth_headers(): return {"Authorization": f"Bearer {TOKEN}"}response = requests.get(f"{BASE_URL}/v1/synthesis/engines", headers=auth_headers())response.raise_for_status()data = response.json()print(data)const BASE_URL = process.env.CORTEX_URL ?? "http://127.0.0.1:8080";const TOKEN = process.env.CORTEX_TOKEN ?? "replace_with_token";const authHeaders = { Authorization: `Bearer ${TOKEN}`,};const response = await fetch(`${BASE_URL}/v1/synthesis/engines`, { headers: authHeaders,});if (!response.ok) throw new Error(await response.text());const data = await response.json();console.log(data);import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;public class CortexExample { static final String BASE_URL = System.getenv().getOrDefault("CORTEX_URL", "http://127.0.0.1:8080"); static final String TOKEN = System.getenv().getOrDefault("CORTEX_TOKEN", "replace_with_token"); static final HttpClient HTTP = HttpClient.newHttpClient(); static void print(HttpResponse<String> response) { System.out.println(response.statusCode()); System.out.println(response.body()); } public static void main(String[] args) throws Exception { HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + "/v1/synthesis/engines")) .header("Authorization", "Bearer " + TOKEN) .GET() .build(); print(HTTP.send(request, HttpResponse.BodyHandlers.ofString())); }}生成 RAG goldens
import osimport requestsBASE_URL = os.getenv("CORTEX_URL", "http://127.0.0.1:8080")TOKEN = os.getenv("CORTEX_TOKEN", "replace_with_token")def auth_headers(): return {"Authorization": f"Bearer {TOKEN}"}payload = { "name": "quickstart-qa-preview", "synthesis_type": "qa_pairs", "engine_id": "deepeval", "source": { "type": "documents", "documents": [ "Cortex Parse turns URLs and storage objects into LLM-ready Markdown." ] }, "config": { "sample_count": 2, "max_contexts_per_case": 1, "include_expected_output": True }, "output": { "output_format": "json", "include_preview": True }}response = requests.post( f"{BASE_URL}/v1/synthesis/jobs", headers={**auth_headers(), "Content-Type": "application/json"}, json=payload,)response.raise_for_status()data = response.json()print(data)const BASE_URL = process.env.CORTEX_URL ?? "http://127.0.0.1:8080";const TOKEN = process.env.CORTEX_TOKEN ?? "replace_with_token";const authHeaders = { Authorization: `Bearer ${TOKEN}`,};const payload = { "name": "quickstart-qa-preview", "synthesis_type": "qa_pairs", "engine_id": "deepeval", "source": { "type": "documents", "documents": [ "Cortex Parse turns URLs and storage objects into LLM-ready Markdown." ] }, "config": { "sample_count": 2, "max_contexts_per_case": 1, "include_expected_output": true }, "output": { "output_format": "json", "include_preview": true }};const response = await fetch(`${BASE_URL}/v1/synthesis/jobs`, { method: "POST", headers: { ...authHeaders, "Content-Type": "application/json" }, body: JSON.stringify(payload),});if (!response.ok) throw new Error(await response.text());const data = await response.json();console.log(data);import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;public class CortexExample { static final String BASE_URL = System.getenv().getOrDefault("CORTEX_URL", "http://127.0.0.1:8080"); static final String TOKEN = System.getenv().getOrDefault("CORTEX_TOKEN", "replace_with_token"); static final HttpClient HTTP = HttpClient.newHttpClient(); static void print(HttpResponse<String> response) { System.out.println(response.statusCode()); System.out.println(response.body()); } public static void main(String[] args) throws Exception { String json = """ { \"name\": \"quickstart-qa-preview\", \"synthesis_type\": \"qa_pairs\", \"engine_id\": \"deepeval\", \"source\": { \"type\": \"documents\", \"documents\": [ \"Cortex Parse turns URLs and storage objects into LLM-ready Markdown.\" ] }, \"config\": { \"sample_count\": 2, \"max_contexts_per_case\": 1, \"include_expected_output\": true }, \"output\": { \"output_format\": \"json\", \"include_preview\": true } } """; HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + "/v1/synthesis/jobs")) .header("Authorization", "Bearer " + TOKEN) .header("Content-Type", "application/json") .POST(HttpRequest.BodyPublishers.ofString(json)) .build(); print(HTTP.send(request, HttpResponse.BodyHandlers.ofString())); }}获取结果：
import osimport requestsBASE_URL = os.getenv("CORTEX_URL", "http://127.0.0.1:8080")TOKEN = os.getenv("CORTEX_TOKEN", "replace_with_token")def auth_headers(): return {"Authorization": f"Bearer {TOKEN}"}response = requests.get(f"{BASE_URL}/v1/synthesis/jobs/job_xxx/result", headers=auth_headers())response.raise_for_status()data = response.json()print(data)const BASE_URL = process.env.CORTEX_URL ?? "http://127.0.0.1:8080";const TOKEN = process.env.CORTEX_TOKEN ?? "replace_with_token";const authHeaders = { Authorization: `Bearer ${TOKEN}`,};const response = await fetch(`${BASE_URL}/v1/synthesis/jobs/job_xxx/result`, { headers: authHeaders,});if (!response.ok) throw new Error(await response.text());const data = await response.json();console.log(data);import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;public class CortexExample { static final String BASE_URL = System.getenv().getOrDefault("CORTEX_URL", "http://127.0.0.1:8080"); static final String TOKEN = System.getenv().getOrDefault("CORTEX_TOKEN", "replace_with_token"); static final HttpClient HTTP = HttpClient.newHttpClient(); static void print(HttpResponse<String> response) { System.out.println(response.statusCode()); System.out.println(response.body()); } public static void main(String[] args) throws Exception { HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + "/v1/synthesis/jobs/job_xxx/result")) .header("Authorization", "Bearer " + TOKEN) .GET() .build(); print(HTTP.send(request, HttpResponse.BodyHandlers.ofString())); }}异步合成完成后可以持久化 synthesis_output 对象，并回写 synthesis_runs.output_object_id。
供应商提示
- Evaluation 和 Synthesis 适配器应接收完整供应商槽位配置。
- 本地 runtime profile 可将 DeepEval 路由到 Kimi、OpenRouter、OpenAI-compatible endpoint 或其他配置供应商。
- 长耗时模型 Job 应使用大于预期 provider latency 的 lease；当前本地示例使用 300 秒 lease 和 30 秒 heartbeat。