{ "version": "1.0", "timestamp": "2026-08-25T17:44:25.622293", "run_id": "20260825-174425", "environment": { "python_version": "3.13.11", "platform": "linux", "model": "qwen3.6:35b-a3b" }, "config": { "cases_requested": [ "case_01_generation", "case_02_simple_tool", "case_03_file_analysis", "case_04_parallel_tool", "case_05_long_context", "case_06_reasoning", "case_07_multiturn" ], "repeats": 2, "skip_heavy": false, "total_cases": 7 }, "summary": { "total_cases": 7, "passed": 7, "failed": 0, "stopped": false, "total_elapsed_seconds": 728.71 }, "results": [ { "case_id": "case_01_generation", "case_name": "纯文本生成基准", "difficulty": "简单", "status": "passed", "raw_data": { "case_id": "case_01_generation", "name": "纯文本生成基准", "description": "基础生成速度基准测试", "difficulty": "简单", "estimated_seconds": 5, "results": [ { "iteration": 1, "ttft_ms": 14715.57, "prefill_ms": 328.38, "decode_speed_tok_s": 80.65, "total_tokens": 331, "prompt_tokens": 31, "completion_tokens": 300, "e2e_ms": 18438.39, "elapsed_ms": 18438.39, "response_length": 912 }, { "iteration": 2, "ttft_ms": 13780.79, "prefill_ms": 338.35, "decode_speed_tok_s": 79.88, "total_tokens": 331, "prompt_tokens": 31, "completion_tokens": 300, "e2e_ms": 17539.77, "elapsed_ms": 17539.77, "response_length": 988 } ], "prompt_used": "请用200字左右概述人工智能从诞生至今的关键发展阶段，每段不超过50字。" }, "statistics": { "count": 2, "mean_ms": 17989.08, "std_ms": 635.42, "min_ms": 17539.77, "max_ms": 18438.39, "p50_ms": 17989.08, "p95_ms": 18393.46, "p99_ms": 18429.4 }, "inference_metrics": { "ttft_ms_mean": 14248.18, "ttft_ms_p95": 14668.83, "prefill_ms_mean": 333.37, "prefill_ms_p95": 337.85, "decode_speed_tok_s_mean": 80.27, "decode_speed_tok_s_p95": 80.61, "e2e_ms_mean": 17989.08, "e2e_ms_p95": 18393.46, "total_tokens_mean": 331.0, "total_tokens_p95": 331.0 }, "elapsed_seconds": 35.98 }, { "case_id": "case_02_simple_tool", "case_name": "简单工具调用延迟", "difficulty": "简单", "status": "passed", "raw_data": { "case_id": "case_02_simple_tool", "name": "简单工具调用延迟", "description": "测量简单工具调用的额外延迟", "difficulty": "简单", "estimated_seconds": 3, "results": [ { "iteration": 1, "ttft_ms": 16196.0, "prefill_ms": 282.79, "decode_speed_tok_s": 84.08, "total_tokens": 83, "prompt_tokens": 61, "completion_tokens": 22, "e2e_ms": 16459.21, "elapsed_ms": 16459.21, "response_length": 60, "tool": "get_date", "matched": false }, { "iteration": 2, "ttft_ms": 14408.31, "prefill_ms": 289.76, "decode_speed_tok_s": 81.91, "total_tokens": 122, "prompt_tokens": 62, "completion_tokens": 60, "e2e_ms": 15142.91, "elapsed_ms": 15142.91, "response_length": 117, "tool": "get_cpu_usage", "matched": false } ] }, "statistics": { "count": 2, "mean_ms": 15801.06, "std_ms": 930.76, "min_ms": 15142.91, "max_ms": 16459.21, "p50_ms": 15801.06, "p95_ms": 16393.4, "p99_ms": 16446.05 }, "inference_metrics": { "ttft_ms_mean": 15302.15, "ttft_ms_p95": 16106.62, "prefill_ms_mean": 286.27, "prefill_ms_p95": 289.41, "decode_speed_tok_s_mean": 83.0, "decode_speed_tok_s_p95": 83.97, "e2e_ms_mean": 15801.06, "e2e_ms_p95": 16393.4, "total_tokens_mean": 102.5, "total_tokens_p95": 120.05 }, "elapsed_seconds": 31.6 }, { "case_id": "case_03_file_analysis", "case_name": "文件读取+分析", "difficulty": "中等", "status": "passed", "raw_data": { "case_id": "case_03_file_analysis", "name": "文件读取+分析", "description": "文件读取与分析完整链路计时", "difficulty": "中等", "estimated_seconds": 2, "results": [ { "iteration": 1, "ttft_ms": 4497.2, "prefill_ms": 96.54, "decode_speed_tok_s": 81.86, "total_tokens": 485, "prompt_tokens": 185, "completion_tokens": 300, "e2e_ms": 8166.59, "elapsed_ms": 8166.59, "response_length": 1122, "file_chars": 485 }, { "iteration": 2, "ttft_ms": 30717.41, "prefill_ms": 103.84, "decode_speed_tok_s": 79.84, "total_tokens": 485, "prompt_tokens": 185, "completion_tokens": 300, "e2e_ms": 34478.78, "elapsed_ms": 34478.78, "response_length": 1075, "file_chars": 485 } ] }, "statistics": { "count": 2, "mean_ms": 21322.68, "std_ms": 18605.53, "min_ms": 8166.59, "max_ms": 34478.78, "p50_ms": 21322.68, "p95_ms": 33163.17, "p99_ms": 34215.66 }, "inference_metrics": { "ttft_ms_mean": 17607.31, "ttft_ms_p95": 29406.4, "prefill_ms_mean": 100.19, "prefill_ms_p95": 103.47, "decode_speed_tok_s_mean": 80.85, "decode_speed_tok_s_p95": 81.76, "e2e_ms_mean": 21322.68, "e2e_ms_p95": 33163.17, "total_tokens_mean": 485.0, "total_tokens_p95": 485.0 }, "elapsed_seconds": 42.65 }, { "case_id": "case_04_parallel_tool", "case_name": "并行工具调用", "difficulty": "中等", "status": "passed", "raw_data": { "case_id": "case_04_parallel_tool", "name": "并行工具调用", "description": "对比串行 vs 并行工具调用的延迟差异", "difficulty": "中等", "estimated_seconds": 5, "results": [ { "iteration": 1, "parallel_tasks": 3, "serial_ms": 37309.93, "parallel_ms": 45353.04, "speedup_ratio": 0.82, "elapsed_ms": 45353.04, "avg_ttft_ms": 43165.98, "avg_decode_speed_tok_s": 83.01, "ttft_ms": 43165.98, "decode_speed_tok_s": 83.01 }, { "iteration": 2, "parallel_tasks": 3, "serial_ms": 30577.28, "parallel_ms": 10193.83, "speedup_ratio": 3.0, "elapsed_ms": 10193.83, "avg_ttft_ms": 7917.49, "avg_decode_speed_tok_s": 81.22, "ttft_ms": 7917.49, "decode_speed_tok_s": 81.22 } ] }, "statistics": { "count": 2, "mean_ms": 27773.44, "std_ms": 24861.32, "min_ms": 10193.83, "max_ms": 45353.04, "p50_ms": 27773.44, "p95_ms": 43595.08, "p99_ms": 45001.45 }, "inference_metrics": { "ttft_ms_mean": 25541.74, "ttft_ms_p95": 41403.56, "decode_speed_tok_s_mean": 82.12, "decode_speed_tok_s_p95": 82.92 }, "elapsed_seconds": 123.43 }, { "case_id": "case_05_long_context", "case_name": "长上下文处理", "difficulty": "耗时", "status": "passed", "raw_data": { "case_id": "case_05_long_context", "name": "长上下文处理", "description": "不同上下文长度对延迟的影响", "difficulty": "耗时", "estimated_seconds": 10, "results": [ { "iteration": 1, "ttft_ms": 5375.1, "prefill_ms": 842.82, "decode_speed_tok_s": 80.26, "total_tokens": 728, "prompt_tokens": 648, "completion_tokens": 80, "e2e_ms": 6378.84, "elapsed_ms": 6378.84, "response_length": 319, "context_tokens_target": 500, "context_chars": 1254 }, { "iteration": 2, "ttft_ms": 57883.93, "prefill_ms": 95.78, "decode_speed_tok_s": 83.77, "total_tokens": 728, "prompt_tokens": 648, "completion_tokens": 80, "e2e_ms": 58844.25, "elapsed_ms": 58844.25, "response_length": 319, "context_tokens_target": 500, "context_chars": 1254 }, { "iteration": 1, "ttft_ms": 14493.28, "prefill_ms": 1041.01, "decode_speed_tok_s": 81.17, "total_tokens": 1355, "prompt_tokens": 1275, "completion_tokens": 80, "e2e_ms": 15491.6, "elapsed_ms": 15491.6, "response_length": 315, "context_tokens_target": 1000, "context_chars": 2508 }, { "iteration": 2, "ttft_ms": 88553.55, "prefill_ms": 102.99, "decode_speed_tok_s": 82.75, "total_tokens": 1355, "prompt_tokens": 1275, "completion_tokens": 80, "e2e_ms": 89532.55, "elapsed_ms": 89532.55, "response_length": 319, "context_tokens_target": 1000, "context_chars": 2508 }, { "iteration": 1, "ttft_ms": 19499.27, "prefill_ms": 1513.23, "decode_speed_tok_s": 80.77, "total_tokens": 2628, "prompt_tokens": 2548, "completion_tokens": 80, "e2e_ms": 20512.64, "elapsed_ms": 20512.64, "response_length": 331, "context_tokens_target": 2000, "context_chars": 5054 }, { "iteration": 2, "ttft_ms": 11929.32, "prefill_ms": 114.78, "decode_speed_tok_s": 77.52, "total_tokens": 2628, "prompt_tokens": 2548, "completion_tokens": 80, "e2e_ms": 12983.72, "elapsed_ms": 12983.72, "response_length": 329, "context_tokens_target": 2000, "context_chars": 5054 }, { "iteration": 1, "ttft_ms": 14999.33, "prefill_ms": 3185.49, "decode_speed_tok_s": 79.84, "total_tokens": 5155, "prompt_tokens": 5075, "completion_tokens": 80, "e2e_ms": 16042.1, "elapsed_ms": 16042.1, "response_length": 329, "context_tokens_target": 4000, "context_chars": 10108 }, { "iteration": 2, "ttft_ms": 9751.38, "prefill_ms": 116.54, "decode_speed_tok_s": 83.8, "total_tokens": 5155, "prompt_tokens": 5075, "completion_tokens": 80, "e2e_ms": 10748.62, "elapsed_ms": 10748.62, "response_length": 331, "context_tokens_target": 4000, "context_chars": 10108 } ], "context_sizes_tested": [ 500, 1000, 2000, 4000 ] }, "statistics": { "count": 8, "mean_ms": 28816.79, "std_ms": 29467.59, "min_ms": 6378.84, "max_ms": 89532.55, "p50_ms": 15766.85, "p95_ms": 78791.64, "p99_ms": 87384.37 }, "inference_metrics": { "ttft_ms_mean": 27810.65, "ttft_ms_p95": 77819.18, "prefill_ms_mean": 876.58, "prefill_ms_p95": 2600.2, "decode_speed_tok_s_mean": 81.23, "decode_speed_tok_s_p95": 83.79, "e2e_ms_mean": 28816.79, "e2e_ms_p95": 78791.64, "total_tokens_mean": 2466.5, "total_tokens_p95": 5155.0 }, "elapsed_seconds": 230.54 }, { "case_id": "case_06_reasoning", "case_name": "复杂推理任务", "difficulty": "耗时", "status": "passed", "raw_data": { "case_id": "case_06_reasoning", "name": "复杂推理任务", "description": "不同推理复杂度下的延迟对比", "difficulty": "耗时", "estimated_seconds": 15, "results": [ { "iteration": 1, "ttft_ms": 17139.94, "prefill_ms": 284.82, "decode_speed_tok_s": 81.61, "total_tokens": 93, "prompt_tokens": 13, "completion_tokens": 80, "e2e_ms": 18122.0, "elapsed_ms": 18122.0, "response_length": 314, "reasoning_level": "简单问答" }, { "iteration": 2, "ttft_ms": 47602.11, "prefill_ms": 285.31, "decode_speed_tok_s": 80.04, "total_tokens": 93, "prompt_tokens": 13, "completion_tokens": 80, "e2e_ms": 48603.85, "elapsed_ms": 48603.85, "response_length": 313, "reasoning_level": "简单问答" }, { "iteration": 1, "ttft_ms": 10822.25, "prefill_ms": 315.46, "decode_speed_tok_s": 82.22, "total_tokens": 220, "prompt_tokens": 20, "completion_tokens": 200, "e2e_ms": 13257.81, "elapsed_ms": 13257.81, "response_length": 811, "reasoning_level": "中等推理" }, { "iteration": 2, "ttft_ms": 20657.36, "prefill_ms": 314.18, "decode_speed_tok_s": 80.9, "total_tokens": 220, "prompt_tokens": 20, "completion_tokens": 200, "e2e_ms": 23132.11, "elapsed_ms": 23132.11, "response_length": 745, "reasoning_level": "中等推理" }, { "iteration": 1, "ttft_ms": 8117.27, "prefill_ms": 335.93, "decode_speed_tok_s": 82.11, "total_tokens": 427, "prompt_tokens": 27, "completion_tokens": 400, "e2e_ms": 12993.07, "elapsed_ms": 12993.07, "response_length": 1762, "reasoning_level": "复杂推理" }, { "iteration": 2, "ttft_ms": 6073.91, "prefill_ms": 333.3, "decode_speed_tok_s": 81.79, "total_tokens": 427, "prompt_tokens": 27, "completion_tokens": 400, "e2e_ms": 10969.79, "elapsed_ms": 10969.79, "response_length": 1809, "reasoning_level": "复杂推理" } ] }, "statistics": { "count": 6, "mean_ms": 21179.77, "std_ms": 14136.94, "min_ms": 10969.79, "max_ms": 48603.85, "p50_ms": 15689.9, "p95_ms": 42235.91, "p99_ms": 47330.26 }, "inference_metrics": { "ttft_ms_mean": 18402.14, "ttft_ms_p95": 40865.92, "prefill_ms_mean": 311.5, "prefill_ms_p95": 335.27, "decode_speed_tok_s_mean": 81.45, "decode_speed_tok_s_p95": 82.19, "e2e_ms_mean": 21179.77, "e2e_ms_p95": 42235.91, "total_tokens_mean": 246.67, "total_tokens_p95": 427.0 }, "elapsed_seconds": 127.08 }, { "case_id": "case_07_multiturn", "case_name": "多轮对话累积", "difficulty": "耗时", "status": "passed", "raw_data": { "case_id": "case_07_multiturn", "name": "多轮对话累积", "description": "追踪连续多轮对话中延迟随上下文累积的变化", "difficulty": "耗时", "estimated_seconds": 10, "results": [ { "iteration": 1, "ttft_ms": 352.84, "prefill_ms": 352.84, "decode_speed_tok_s": 81.88, "total_tokens": 153, "prompt_tokens": 33, "completion_tokens": 120, "e2e_ms": 7458.81, "elapsed_ms": 7458.81, "response_length": 0, "round": 1, "repeat": 1, "scenario": "修复Python中的索引越界错误", "messages_in_context": 2 }, { "iteration": 2, "ttft_ms": 570.91, "prefill_ms": 570.91, "decode_speed_tok_s": 83.44, "total_tokens": 186, "prompt_tokens": 66, "completion_tokens": 120, "e2e_ms": 9924.72, "elapsed_ms": 9924.72, "response_length": 0, "round": 2, "repeat": 1, "scenario": "修复JavaScript闭包变量捕获问题", "messages_in_context": 4 }, { "iteration": 3, "ttft_ms": 592.78, "prefill_ms": 592.78, "decode_speed_tok_s": 80.86, "total_tokens": 216, "prompt_tokens": 96, "completion_tokens": 120, "e2e_ms": 8754.91, "elapsed_ms": 8754.91, "response_length": 0, "round": 3, "repeat": 1, "scenario": "修复SQL注入漏洞", "messages_in_context": 6 }, { "iteration": 4, "ttft_ms": 468.43, "prefill_ms": 468.43, "decode_speed_tok_s": 83.91, "total_tokens": 247, "prompt_tokens": 127, "completion_tokens": 120, "e2e_ms": 8867.78, "elapsed_ms": 8867.78, "response_length": 0, "round": 4, "repeat": 1, "scenario": "修复多线程竞态条件", "messages_in_context": 8 }, { "iteration": 5, "ttft_ms": 472.24, "prefill_ms": 472.24, "decode_speed_tok_s": 81.92, "total_tokens": 277, "prompt_tokens": 157, "completion_tokens": 120, "e2e_ms": 9069.2, "elapsed_ms": 9069.2, "response_length": 0, "round": 5, "repeat": 1, "scenario": "修复递归栈溢出", "messages_in_context": 10 }, { "iteration": 6, "ttft_ms": 496.68, "prefill_ms": 496.68, "decode_speed_tok_s": 83.06, "total_tokens": 308, "prompt_tokens": 188, "completion_tokens": 120, "e2e_ms": 9110.29, "elapsed_ms": 9110.29, "response_length": 0, "round": 6, "repeat": 1, "scenario": "修复正则表达式灾难回溯", "messages_in_context": 12 }, { "iteration": 7, "ttft_ms": 356.02, "prefill_ms": 356.02, "decode_speed_tok_s": 82.34, "total_tokens": 153, "prompt_tokens": 33, "completion_tokens": 120, "e2e_ms": 9302.89, "elapsed_ms": 9302.89, "response_length": 0, "round": 1, "repeat": 2, "scenario": "修复Python中的索引越界错误", "messages_in_context": 2 }, { "iteration": 8, "ttft_ms": 105.61, "prefill_ms": 105.61, "decode_speed_tok_s": 80.39, "total_tokens": 186, "prompt_tokens": 66, "completion_tokens": 120, "e2e_ms": 9569.96, "elapsed_ms": 9569.96, "response_length": 0, "round": 2, "repeat": 2, "scenario": "修复JavaScript闭包变量捕获问题", "messages_in_context": 4 }, { "iteration": 9, "ttft_ms": 596.24, "prefill_ms": 596.24, "decode_speed_tok_s": 78.49, "total_tokens": 216, "prompt_tokens": 96, "completion_tokens": 120, "e2e_ms": 9446.57, "elapsed_ms": 9446.57, "response_length": 0, "round": 3, "repeat": 2, "scenario": "修复SQL注入漏洞", "messages_in_context": 6 }, { "iteration": 10, "ttft_ms": 468.0, "prefill_ms": 468.0, "decode_speed_tok_s": 81.77, "total_tokens": 247, "prompt_tokens": 127, "completion_tokens": 120, "e2e_ms": 7437.1, "elapsed_ms": 7437.1, "response_length": 0, "round": 4, "repeat": 2, "scenario": "修复多线程竞态条件", "messages_in_context": 8 }, { "iteration": 11, "ttft_ms": 470.8, "prefill_ms": 470.8, "decode_speed_tok_s": 81.38, "total_tokens": 277, "prompt_tokens": 157, "completion_tokens": 120, "e2e_ms": 22114.83, "elapsed_ms": 22114.83, "response_length": 0, "round": 5, "repeat": 2, "scenario": "修复递归栈溢出", "messages_in_context": 10 }, { "iteration": 12, "ttft_ms": 503.16, "prefill_ms": 503.16, "decode_speed_tok_s": 81.86, "total_tokens": 308, "prompt_tokens": 188, "completion_tokens": 120, "e2e_ms": 26371.35, "elapsed_ms": 26371.35, "response_length": 0, "round": 6, "repeat": 2, "scenario": "修复正则表达式灾难回溯", "messages_in_context": 12 } ], "total_rounds": 12 }, "statistics": { "count": 12, "mean_ms": 11452.37, "std_ms": 6090.06, "min_ms": 7437.1, "max_ms": 26371.35, "p50_ms": 9206.59, "p95_ms": 24030.26, "p99_ms": 25903.13 }, "inference_metrics": { "ttft_ms_mean": 454.48, "ttft_ms_p95": 594.34, "prefill_ms_mean": 454.48, "prefill_ms_p95": 594.34, "decode_speed_tok_s_mean": 81.77, "decode_speed_tok_s_p95": 83.65, "e2e_ms_mean": 11452.37, "e2e_ms_p95": 24030.26, "total_tokens_mean": 231.17, "total_tokens_p95": 308.0 }, "elapsed_seconds": 137.43 } ] }