562 lines
16 KiB
JSON
562 lines
16 KiB
JSON
{
|
|
"version": "1.0",
|
|
"timestamp": "2026-08-25T17:25:32.979324",
|
|
"run_id": "20260825-172532",
|
|
"environment": {
|
|
"python_version": "3.13.11",
|
|
"platform": "linux",
|
|
"model": "qwen3.6:35b-a3b"
|
|
},
|
|
"config": {
|
|
"cases_requested": [
|
|
"case_01_generation",
|
|
"case_02_simple_tool",
|
|
"case_03_file_analysis",
|
|
"case_04_parallel_tool",
|
|
"case_05_long_context",
|
|
"case_06_reasoning",
|
|
"case_07_multiturn"
|
|
],
|
|
"repeats": 2,
|
|
"skip_heavy": false,
|
|
"total_cases": 7
|
|
},
|
|
"summary": {
|
|
"total_cases": 7,
|
|
"passed": 6,
|
|
"failed": 1,
|
|
"stopped": false,
|
|
"total_elapsed_seconds": 357.49
|
|
},
|
|
"results": [
|
|
{
|
|
"case_id": "case_01_generation",
|
|
"case_name": "纯文本生成基准",
|
|
"difficulty": "简单",
|
|
"status": "passed",
|
|
"raw_data": {
|
|
"case_id": "case_01_generation",
|
|
"name": "纯文本生成基准",
|
|
"description": "基础生成速度基准测试",
|
|
"difficulty": "简单",
|
|
"estimated_seconds": 5,
|
|
"results": [
|
|
{
|
|
"iteration": 1,
|
|
"ttft_ms": 41583.27,
|
|
"prefill_ms": 319.34,
|
|
"decode_speed_tok_s": 76.72,
|
|
"total_tokens": 331,
|
|
"prompt_tokens": 31,
|
|
"completion_tokens": 300,
|
|
"e2e_ms": 45496.03,
|
|
"elapsed_ms": 45496.03,
|
|
"response_length": 969
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"ttft_ms": 8929.4,
|
|
"prefill_ms": 315.75,
|
|
"decode_speed_tok_s": 28.7,
|
|
"total_tokens": 331,
|
|
"prompt_tokens": 31,
|
|
"completion_tokens": 300,
|
|
"e2e_ms": 19385.96,
|
|
"elapsed_ms": 19385.96,
|
|
"response_length": 961
|
|
}
|
|
],
|
|
"prompt_used": "请用200字左右概述人工智能从诞生至今的关键发展阶段，每段不超过50字。"
|
|
},
|
|
"statistics": {
|
|
"count": 2,
|
|
"mean_ms": 32440.99,
|
|
"std_ms": 18462.61,
|
|
"min_ms": 19385.96,
|
|
"max_ms": 45496.03,
|
|
"p50_ms": 32440.99,
|
|
"p95_ms": 44190.53,
|
|
"p99_ms": 45234.93
|
|
},
|
|
"inference_metrics": {
|
|
"ttft_ms_mean": 25256.33,
|
|
"ttft_ms_p95": 39950.58,
|
|
"prefill_ms_mean": 317.54,
|
|
"prefill_ms_p95": 319.16,
|
|
"decode_speed_tok_s_mean": 52.71,
|
|
"decode_speed_tok_s_p95": 74.32,
|
|
"e2e_ms_mean": 32440.99,
|
|
"e2e_ms_p95": 44190.53,
|
|
"total_tokens_mean": 331.0,
|
|
"total_tokens_p95": 331.0
|
|
},
|
|
"elapsed_seconds": 64.88
|
|
},
|
|
{
|
|
"case_id": "case_02_simple_tool",
|
|
"case_name": "简单工具调用延迟",
|
|
"difficulty": "简单",
|
|
"status": "passed",
|
|
"raw_data": {
|
|
"case_id": "case_02_simple_tool",
|
|
"name": "简单工具调用延迟",
|
|
"description": "测量简单工具调用的额外延迟",
|
|
"difficulty": "简单",
|
|
"estimated_seconds": 3,
|
|
"results": [
|
|
{
|
|
"iteration": 1,
|
|
"ttft_ms": 22079.81,
|
|
"prefill_ms": 317.66,
|
|
"decode_speed_tok_s": 77.75,
|
|
"total_tokens": 121,
|
|
"prompt_tokens": 61,
|
|
"completion_tokens": 60,
|
|
"e2e_ms": 22854.03,
|
|
"elapsed_ms": 22854.03,
|
|
"response_length": 117,
|
|
"tool": "get_date",
|
|
"matched": false
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"ttft_ms": 68555.6,
|
|
"prefill_ms": 368.48,
|
|
"decode_speed_tok_s": 82.77,
|
|
"total_tokens": 122,
|
|
"prompt_tokens": 62,
|
|
"completion_tokens": 60,
|
|
"e2e_ms": 69282.23,
|
|
"elapsed_ms": 69282.23,
|
|
"response_length": 118,
|
|
"tool": "get_cpu_usage",
|
|
"matched": false
|
|
}
|
|
]
|
|
},
|
|
"statistics": {
|
|
"count": 2,
|
|
"mean_ms": 46068.13,
|
|
"std_ms": 32829.7,
|
|
"min_ms": 22854.03,
|
|
"max_ms": 69282.23,
|
|
"p50_ms": 46068.13,
|
|
"p95_ms": 66960.82,
|
|
"p99_ms": 68817.95
|
|
},
|
|
"inference_metrics": {
|
|
"ttft_ms_mean": 45317.71,
|
|
"ttft_ms_p95": 66231.81,
|
|
"prefill_ms_mean": 343.07,
|
|
"prefill_ms_p95": 365.94,
|
|
"decode_speed_tok_s_mean": 80.26,
|
|
"decode_speed_tok_s_p95": 82.52,
|
|
"e2e_ms_mean": 46068.13,
|
|
"e2e_ms_p95": 66960.82,
|
|
"total_tokens_mean": 121.5,
|
|
"total_tokens_p95": 121.95
|
|
},
|
|
"elapsed_seconds": 92.14
|
|
},
|
|
{
|
|
"case_id": "case_03_file_analysis",
|
|
"case_name": "文件读取+分析",
|
|
"difficulty": "中等",
|
|
"status": "passed",
|
|
"raw_data": {
|
|
"case_id": "case_03_file_analysis",
|
|
"name": "文件读取+分析",
|
|
"description": "文件读取与分析完整链路计时",
|
|
"difficulty": "中等",
|
|
"estimated_seconds": 2,
|
|
"results": [
|
|
{
|
|
"iteration": 1,
|
|
"ttft_ms": 72428.05,
|
|
"prefill_ms": 643.53,
|
|
"decode_speed_tok_s": 81.45,
|
|
"total_tokens": 485,
|
|
"prompt_tokens": 185,
|
|
"completion_tokens": 300,
|
|
"e2e_ms": 76116.09,
|
|
"elapsed_ms": 76116.09,
|
|
"response_length": 1083,
|
|
"file_chars": 485
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"ttft_ms": 233.37,
|
|
"prefill_ms": 103.1,
|
|
"decode_speed_tok_s": 84.74,
|
|
"total_tokens": 485,
|
|
"prompt_tokens": 185,
|
|
"completion_tokens": 300,
|
|
"e2e_ms": 3778.0,
|
|
"elapsed_ms": 3778.0,
|
|
"response_length": 1111,
|
|
"file_chars": 485
|
|
}
|
|
]
|
|
},
|
|
"statistics": {
|
|
"count": 2,
|
|
"mean_ms": 39947.04,
|
|
"std_ms": 51150.75,
|
|
"min_ms": 3778.0,
|
|
"max_ms": 76116.09,
|
|
"p50_ms": 39947.04,
|
|
"p95_ms": 72499.19,
|
|
"p99_ms": 75392.71
|
|
},
|
|
"inference_metrics": {
|
|
"ttft_ms_mean": 36330.71,
|
|
"ttft_ms_p95": 68818.32,
|
|
"prefill_ms_mean": 373.31,
|
|
"prefill_ms_p95": 616.51,
|
|
"decode_speed_tok_s_mean": 83.09,
|
|
"decode_speed_tok_s_p95": 84.58,
|
|
"e2e_ms_mean": 39947.04,
|
|
"e2e_ms_p95": 72499.19,
|
|
"total_tokens_mean": 485.0,
|
|
"total_tokens_p95": 485.0
|
|
},
|
|
"elapsed_seconds": 79.9
|
|
},
|
|
{
|
|
"case_id": "case_04_parallel_tool",
|
|
"case_name": "并行工具调用",
|
|
"difficulty": "中等",
|
|
"status": "passed",
|
|
"raw_data": {
|
|
"case_id": "case_04_parallel_tool",
|
|
"name": "并行工具调用",
|
|
"description": "对比串行 vs 并行工具调用的延迟差异",
|
|
"difficulty": "中等",
|
|
"estimated_seconds": 5,
|
|
"results": [
|
|
{
|
|
"iteration": 1,
|
|
"parallel_tasks": 3,
|
|
"serial_ms": 4045.91,
|
|
"parallel_ms": 3855.57,
|
|
"speedup_ratio": 1.05,
|
|
"elapsed_ms": 3855.57,
|
|
"avg_ttft_ms": 1662.77,
|
|
"avg_decode_speed_tok_s": 84.11,
|
|
"ttft_ms": 1662.77,
|
|
"decode_speed_tok_s": 84.11
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"parallel_tasks": 3,
|
|
"serial_ms": 4303.19,
|
|
"parallel_ms": 3989.56,
|
|
"speedup_ratio": 1.08,
|
|
"elapsed_ms": 3989.56,
|
|
"avg_ttft_ms": 1715.28,
|
|
"avg_decode_speed_tok_s": 84.17,
|
|
"ttft_ms": 1715.28,
|
|
"decode_speed_tok_s": 84.17
|
|
}
|
|
]
|
|
},
|
|
"statistics": {
|
|
"count": 2,
|
|
"mean_ms": 3922.57,
|
|
"std_ms": 94.75,
|
|
"min_ms": 3855.57,
|
|
"max_ms": 3989.56,
|
|
"p50_ms": 3922.57,
|
|
"p95_ms": 3982.86,
|
|
"p99_ms": 3988.22
|
|
},
|
|
"inference_metrics": {
|
|
"ttft_ms_mean": 1689.03,
|
|
"ttft_ms_p95": 1712.65,
|
|
"decode_speed_tok_s_mean": 84.14,
|
|
"decode_speed_tok_s_p95": 84.17
|
|
},
|
|
"elapsed_seconds": 16.19
|
|
},
|
|
{
|
|
"case_id": "case_05_long_context",
|
|
"case_name": "长上下文处理",
|
|
"difficulty": "耗时",
|
|
"status": "passed",
|
|
"raw_data": {
|
|
"case_id": "case_05_long_context",
|
|
"name": "长上下文处理",
|
|
"description": "不同上下文长度对延迟的影响",
|
|
"difficulty": "耗时",
|
|
"estimated_seconds": 10,
|
|
"results": [
|
|
{
|
|
"iteration": 1,
|
|
"ttft_ms": 820.58,
|
|
"prefill_ms": 804.03,
|
|
"decode_speed_tok_s": 77.15,
|
|
"total_tokens": 728,
|
|
"prompt_tokens": 648,
|
|
"completion_tokens": 80,
|
|
"e2e_ms": 1866.99,
|
|
"elapsed_ms": 1866.99,
|
|
"response_length": 319,
|
|
"context_tokens_target": 500,
|
|
"context_chars": 1254
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"ttft_ms": 70076.71,
|
|
"prefill_ms": 97.84,
|
|
"decode_speed_tok_s": 77.76,
|
|
"total_tokens": 728,
|
|
"prompt_tokens": 648,
|
|
"completion_tokens": 80,
|
|
"e2e_ms": 71112.83,
|
|
"elapsed_ms": 71112.83,
|
|
"response_length": 319,
|
|
"context_tokens_target": 500,
|
|
"context_chars": 1254
|
|
},
|
|
{
|
|
"iteration": 1,
|
|
"ttft_ms": 1050.97,
|
|
"prefill_ms": 1030.17,
|
|
"decode_speed_tok_s": 82.71,
|
|
"total_tokens": 1355,
|
|
"prompt_tokens": 1275,
|
|
"completion_tokens": 80,
|
|
"e2e_ms": 2030.18,
|
|
"elapsed_ms": 2030.18,
|
|
"response_length": 319,
|
|
"context_tokens_target": 1000,
|
|
"context_chars": 2508
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"ttft_ms": 121.85,
|
|
"prefill_ms": 100.81,
|
|
"decode_speed_tok_s": 81.98,
|
|
"total_tokens": 1355,
|
|
"prompt_tokens": 1275,
|
|
"completion_tokens": 80,
|
|
"e2e_ms": 1109.55,
|
|
"elapsed_ms": 1109.55,
|
|
"response_length": 319,
|
|
"context_tokens_target": 1000,
|
|
"context_chars": 2508
|
|
},
|
|
{
|
|
"iteration": 1,
|
|
"ttft_ms": 2106.57,
|
|
"prefill_ms": 2077.92,
|
|
"decode_speed_tok_s": 79.78,
|
|
"total_tokens": 2628,
|
|
"prompt_tokens": 2548,
|
|
"completion_tokens": 80,
|
|
"e2e_ms": 3132.13,
|
|
"elapsed_ms": 3132.13,
|
|
"response_length": 331,
|
|
"context_tokens_target": 2000,
|
|
"context_chars": 5054
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"ttft_ms": 133.36,
|
|
"prefill_ms": 105.11,
|
|
"decode_speed_tok_s": 81.07,
|
|
"total_tokens": 2628,
|
|
"prompt_tokens": 2548,
|
|
"completion_tokens": 80,
|
|
"e2e_ms": 1143.05,
|
|
"elapsed_ms": 1143.05,
|
|
"response_length": 331,
|
|
"context_tokens_target": 2000,
|
|
"context_chars": 5054
|
|
},
|
|
{
|
|
"iteration": 1,
|
|
"ttft_ms": 3200.13,
|
|
"prefill_ms": 3153.14,
|
|
"decode_speed_tok_s": 78.66,
|
|
"total_tokens": 5155,
|
|
"prompt_tokens": 5075,
|
|
"completion_tokens": 80,
|
|
"e2e_ms": 4259.59,
|
|
"elapsed_ms": 4259.59,
|
|
"response_length": 331,
|
|
"context_tokens_target": 4000,
|
|
"context_chars": 10108
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"ttft_ms": 165.67,
|
|
"prefill_ms": 117.35,
|
|
"decode_speed_tok_s": 79.5,
|
|
"total_tokens": 5155,
|
|
"prompt_tokens": 5075,
|
|
"completion_tokens": 80,
|
|
"e2e_ms": 1210.27,
|
|
"elapsed_ms": 1210.27,
|
|
"response_length": 326,
|
|
"context_tokens_target": 4000,
|
|
"context_chars": 10108
|
|
}
|
|
],
|
|
"context_sizes_tested": [
|
|
500,
|
|
1000,
|
|
2000,
|
|
4000
|
|
]
|
|
},
|
|
"statistics": {
|
|
"count": 8,
|
|
"mean_ms": 10733.07,
|
|
"std_ms": 24421.91,
|
|
"min_ms": 1109.55,
|
|
"max_ms": 71112.83,
|
|
"p50_ms": 1948.59,
|
|
"p95_ms": 47714.2,
|
|
"p99_ms": 66433.1
|
|
},
|
|
"inference_metrics": {
|
|
"ttft_ms_mean": 9709.48,
|
|
"ttft_ms_p95": 46669.91,
|
|
"prefill_ms_mean": 935.8,
|
|
"prefill_ms_p95": 2776.81,
|
|
"decode_speed_tok_s_mean": 79.83,
|
|
"decode_speed_tok_s_p95": 82.45,
|
|
"e2e_ms_mean": 10733.07,
|
|
"e2e_ms_p95": 47714.2,
|
|
"total_tokens_mean": 2466.5,
|
|
"total_tokens_p95": 5155.0
|
|
},
|
|
"elapsed_seconds": 85.87
|
|
},
|
|
{
|
|
"case_id": "case_06_reasoning",
|
|
"case_name": "复杂推理任务",
|
|
"difficulty": "耗时",
|
|
"status": "passed",
|
|
"raw_data": {
|
|
"case_id": "case_06_reasoning",
|
|
"name": "复杂推理任务",
|
|
"description": "不同推理复杂度下的延迟对比",
|
|
"difficulty": "耗时",
|
|
"estimated_seconds": 15,
|
|
"results": [
|
|
{
|
|
"iteration": 1,
|
|
"ttft_ms": 502.6,
|
|
"prefill_ms": 245.68,
|
|
"decode_speed_tok_s": 80.34,
|
|
"total_tokens": 93,
|
|
"prompt_tokens": 13,
|
|
"completion_tokens": 80,
|
|
"e2e_ms": 1500.12,
|
|
"elapsed_ms": 1500.12,
|
|
"response_length": 320,
|
|
"reasoning_level": "简单问答"
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"ttft_ms": 220.67,
|
|
"prefill_ms": 100.19,
|
|
"decode_speed_tok_s": 83.2,
|
|
"total_tokens": 93,
|
|
"prompt_tokens": 13,
|
|
"completion_tokens": 80,
|
|
"e2e_ms": 1184.6,
|
|
"elapsed_ms": 1184.6,
|
|
"response_length": 317,
|
|
"reasoning_level": "简单问答"
|
|
},
|
|
{
|
|
"iteration": 1,
|
|
"ttft_ms": 438.71,
|
|
"prefill_ms": 268.76,
|
|
"decode_speed_tok_s": 82.97,
|
|
"total_tokens": 220,
|
|
"prompt_tokens": 20,
|
|
"completion_tokens": 200,
|
|
"e2e_ms": 2852.11,
|
|
"elapsed_ms": 2852.11,
|
|
"response_length": 714,
|
|
"reasoning_level": "中等推理"
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"ttft_ms": 232.27,
|
|
"prefill_ms": 97.91,
|
|
"decode_speed_tok_s": 84.27,
|
|
"total_tokens": 220,
|
|
"prompt_tokens": 20,
|
|
"completion_tokens": 200,
|
|
"e2e_ms": 2609.32,
|
|
"elapsed_ms": 2609.32,
|
|
"response_length": 861,
|
|
"reasoning_level": "中等推理"
|
|
},
|
|
{
|
|
"iteration": 1,
|
|
"ttft_ms": 471.77,
|
|
"prefill_ms": 297.05,
|
|
"decode_speed_tok_s": 83.04,
|
|
"total_tokens": 427,
|
|
"prompt_tokens": 27,
|
|
"completion_tokens": 400,
|
|
"e2e_ms": 5293.99,
|
|
"elapsed_ms": 5293.99,
|
|
"response_length": 1768,
|
|
"reasoning_level": "复杂推理"
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"ttft_ms": 234.98,
|
|
"prefill_ms": 101.34,
|
|
"decode_speed_tok_s": 82.71,
|
|
"total_tokens": 427,
|
|
"prompt_tokens": 27,
|
|
"completion_tokens": 400,
|
|
"e2e_ms": 5076.1,
|
|
"elapsed_ms": 5076.1,
|
|
"response_length": 1833,
|
|
"reasoning_level": "复杂推理"
|
|
}
|
|
]
|
|
},
|
|
"statistics": {
|
|
"count": 6,
|
|
"mean_ms": 3086.04,
|
|
"std_ms": 1746.31,
|
|
"min_ms": 1184.6,
|
|
"max_ms": 5293.99,
|
|
"p50_ms": 2730.72,
|
|
"p95_ms": 5239.52,
|
|
"p99_ms": 5283.1
|
|
},
|
|
"inference_metrics": {
|
|
"ttft_ms_mean": 350.17,
|
|
"ttft_ms_p95": 494.89,
|
|
"prefill_ms_mean": 185.16,
|
|
"prefill_ms_p95": 289.98,
|
|
"decode_speed_tok_s_mean": 82.75,
|
|
"decode_speed_tok_s_p95": 84.0,
|
|
"e2e_ms_mean": 3086.04,
|
|
"e2e_ms_p95": 5239.52,
|
|
"total_tokens_mean": 246.67,
|
|
"total_tokens_p95": 427.0
|
|
},
|
|
"elapsed_seconds": 18.52
|
|
},
|
|
{
|
|
"case_id": "case_07_multiturn",
|
|
"case_name": "多轮对话累积",
|
|
"difficulty": "耗时",
|
|
"status": "failed",
|
|
"error": "run_test() got an unexpected keyword argument 'repeats'",
|
|
"elapsed_seconds": 0.0
|
|
}
|
|
]
|
|
} |