{ "version": "1.0", "timestamp": "2026-08-25T17:25:32.979324", "run_id": "20260825-172532", "environment": { "python_version": "3.13.11", "platform": "linux", "model": "qwen3.6:35b-a3b" }, "config": { "cases_requested": [ "case_01_generation", "case_02_simple_tool", "case_03_file_analysis", "case_04_parallel_tool", "case_05_long_context", "case_06_reasoning", "case_07_multiturn" ], "repeats": 2, "skip_heavy": false, "total_cases": 7 }, "summary": { "total_cases": 7, "passed": 6, "failed": 1, "stopped": false, "total_elapsed_seconds": 357.49 }, "results": [ { "case_id": "case_01_generation", "case_name": "纯文本生成基准", "difficulty": "简单", "status": "passed", "raw_data": { "case_id": "case_01_generation", "name": "纯文本生成基准", "description": "基础生成速度基准测试", "difficulty": "简单", "estimated_seconds": 5, "results": [ { "iteration": 1, "ttft_ms": 41583.27, "prefill_ms": 319.34, "decode_speed_tok_s": 76.72, "total_tokens": 331, "prompt_tokens": 31, "completion_tokens": 300, "e2e_ms": 45496.03, "elapsed_ms": 45496.03, "response_length": 969 }, { "iteration": 2, "ttft_ms": 8929.4, "prefill_ms": 315.75, "decode_speed_tok_s": 28.7, "total_tokens": 331, "prompt_tokens": 31, "completion_tokens": 300, "e2e_ms": 19385.96, "elapsed_ms": 19385.96, "response_length": 961 } ], "prompt_used": "请用200字左右概述人工智能从诞生至今的关键发展阶段，每段不超过50字。" }, "statistics": { "count": 2, "mean_ms": 32440.99, "std_ms": 18462.61, "min_ms": 19385.96, "max_ms": 45496.03, "p50_ms": 32440.99, "p95_ms": 44190.53, "p99_ms": 45234.93 }, "inference_metrics": { "ttft_ms_mean": 25256.33, "ttft_ms_p95": 39950.58, "prefill_ms_mean": 317.54, "prefill_ms_p95": 319.16, "decode_speed_tok_s_mean": 52.71, "decode_speed_tok_s_p95": 74.32, "e2e_ms_mean": 32440.99, "e2e_ms_p95": 44190.53, "total_tokens_mean": 331.0, "total_tokens_p95": 331.0 }, "elapsed_seconds": 64.88 }, { "case_id": "case_02_simple_tool", "case_name": "简单工具调用延迟", "difficulty": "简单", "status": "passed", "raw_data": { "case_id": "case_02_simple_tool", "name": "简单工具调用延迟", "description": "测量简单工具调用的额外延迟", "difficulty": "简单", "estimated_seconds": 3, "results": [ { "iteration": 1, "ttft_ms": 22079.81, "prefill_ms": 317.66, "decode_speed_tok_s": 77.75, "total_tokens": 121, "prompt_tokens": 61, "completion_tokens": 60, "e2e_ms": 22854.03, "elapsed_ms": 22854.03, "response_length": 117, "tool": "get_date", "matched": false }, { "iteration": 2, "ttft_ms": 68555.6, "prefill_ms": 368.48, "decode_speed_tok_s": 82.77, "total_tokens": 122, "prompt_tokens": 62, "completion_tokens": 60, "e2e_ms": 69282.23, "elapsed_ms": 69282.23, "response_length": 118, "tool": "get_cpu_usage", "matched": false } ] }, "statistics": { "count": 2, "mean_ms": 46068.13, "std_ms": 32829.7, "min_ms": 22854.03, "max_ms": 69282.23, "p50_ms": 46068.13, "p95_ms": 66960.82, "p99_ms": 68817.95 }, "inference_metrics": { "ttft_ms_mean": 45317.71, "ttft_ms_p95": 66231.81, "prefill_ms_mean": 343.07, "prefill_ms_p95": 365.94, "decode_speed_tok_s_mean": 80.26, "decode_speed_tok_s_p95": 82.52, "e2e_ms_mean": 46068.13, "e2e_ms_p95": 66960.82, "total_tokens_mean": 121.5, "total_tokens_p95": 121.95 }, "elapsed_seconds": 92.14 }, { "case_id": "case_03_file_analysis", "case_name": "文件读取+分析", "difficulty": "中等", "status": "passed", "raw_data": { "case_id": "case_03_file_analysis", "name": "文件读取+分析", "description": "文件读取与分析完整链路计时", "difficulty": "中等", "estimated_seconds": 2, "results": [ { "iteration": 1, "ttft_ms": 72428.05, "prefill_ms": 643.53, "decode_speed_tok_s": 81.45, "total_tokens": 485, "prompt_tokens": 185, "completion_tokens": 300, "e2e_ms": 76116.09, "elapsed_ms": 76116.09, "response_length": 1083, "file_chars": 485 }, { "iteration": 2, "ttft_ms": 233.37, "prefill_ms": 103.1, "decode_speed_tok_s": 84.74, "total_tokens": 485, "prompt_tokens": 185, "completion_tokens": 300, "e2e_ms": 3778.0, "elapsed_ms": 3778.0, "response_length": 1111, "file_chars": 485 } ] }, "statistics": { "count": 2, "mean_ms": 39947.04, "std_ms": 51150.75, "min_ms": 3778.0, "max_ms": 76116.09, "p50_ms": 39947.04, "p95_ms": 72499.19, "p99_ms": 75392.71 }, "inference_metrics": { "ttft_ms_mean": 36330.71, "ttft_ms_p95": 68818.32, "prefill_ms_mean": 373.31, "prefill_ms_p95": 616.51, "decode_speed_tok_s_mean": 83.09, "decode_speed_tok_s_p95": 84.58, "e2e_ms_mean": 39947.04, "e2e_ms_p95": 72499.19, "total_tokens_mean": 485.0, "total_tokens_p95": 485.0 }, "elapsed_seconds": 79.9 }, { "case_id": "case_04_parallel_tool", "case_name": "并行工具调用", "difficulty": "中等", "status": "passed", "raw_data": { "case_id": "case_04_parallel_tool", "name": "并行工具调用", "description": "对比串行 vs 并行工具调用的延迟差异", "difficulty": "中等", "estimated_seconds": 5, "results": [ { "iteration": 1, "parallel_tasks": 3, "serial_ms": 4045.91, "parallel_ms": 3855.57, "speedup_ratio": 1.05, "elapsed_ms": 3855.57, "avg_ttft_ms": 1662.77, "avg_decode_speed_tok_s": 84.11, "ttft_ms": 1662.77, "decode_speed_tok_s": 84.11 }, { "iteration": 2, "parallel_tasks": 3, "serial_ms": 4303.19, "parallel_ms": 3989.56, "speedup_ratio": 1.08, "elapsed_ms": 3989.56, "avg_ttft_ms": 1715.28, "avg_decode_speed_tok_s": 84.17, "ttft_ms": 1715.28, "decode_speed_tok_s": 84.17 } ] }, "statistics": { "count": 2, "mean_ms": 3922.57, "std_ms": 94.75, "min_ms": 3855.57, "max_ms": 3989.56, "p50_ms": 3922.57, "p95_ms": 3982.86, "p99_ms": 3988.22 }, "inference_metrics": { "ttft_ms_mean": 1689.03, "ttft_ms_p95": 1712.65, "decode_speed_tok_s_mean": 84.14, "decode_speed_tok_s_p95": 84.17 }, "elapsed_seconds": 16.19 }, { "case_id": "case_05_long_context", "case_name": "长上下文处理", "difficulty": "耗时", "status": "passed", "raw_data": { "case_id": "case_05_long_context", "name": "长上下文处理", "description": "不同上下文长度对延迟的影响", "difficulty": "耗时", "estimated_seconds": 10, "results": [ { "iteration": 1, "ttft_ms": 820.58, "prefill_ms": 804.03, "decode_speed_tok_s": 77.15, "total_tokens": 728, "prompt_tokens": 648, "completion_tokens": 80, "e2e_ms": 1866.99, "elapsed_ms": 1866.99, "response_length": 319, "context_tokens_target": 500, "context_chars": 1254 }, { "iteration": 2, "ttft_ms": 70076.71, "prefill_ms": 97.84, "decode_speed_tok_s": 77.76, "total_tokens": 728, "prompt_tokens": 648, "completion_tokens": 80, "e2e_ms": 71112.83, "elapsed_ms": 71112.83, "response_length": 319, "context_tokens_target": 500, "context_chars": 1254 }, { "iteration": 1, "ttft_ms": 1050.97, "prefill_ms": 1030.17, "decode_speed_tok_s": 82.71, "total_tokens": 1355, "prompt_tokens": 1275, "completion_tokens": 80, "e2e_ms": 2030.18, "elapsed_ms": 2030.18, "response_length": 319, "context_tokens_target": 1000, "context_chars": 2508 }, { "iteration": 2, "ttft_ms": 121.85, "prefill_ms": 100.81, "decode_speed_tok_s": 81.98, "total_tokens": 1355, "prompt_tokens": 1275, "completion_tokens": 80, "e2e_ms": 1109.55, "elapsed_ms": 1109.55, "response_length": 319, "context_tokens_target": 1000, "context_chars": 2508 }, { "iteration": 1, "ttft_ms": 2106.57, "prefill_ms": 2077.92, "decode_speed_tok_s": 79.78, "total_tokens": 2628, "prompt_tokens": 2548, "completion_tokens": 80, "e2e_ms": 3132.13, "elapsed_ms": 3132.13, "response_length": 331, "context_tokens_target": 2000, "context_chars": 5054 }, { "iteration": 2, "ttft_ms": 133.36, "prefill_ms": 105.11, "decode_speed_tok_s": 81.07, "total_tokens": 2628, "prompt_tokens": 2548, "completion_tokens": 80, "e2e_ms": 1143.05, "elapsed_ms": 1143.05, "response_length": 331, "context_tokens_target": 2000, "context_chars": 5054 }, { "iteration": 1, "ttft_ms": 3200.13, "prefill_ms": 3153.14, "decode_speed_tok_s": 78.66, "total_tokens": 5155, "prompt_tokens": 5075, "completion_tokens": 80, "e2e_ms": 4259.59, "elapsed_ms": 4259.59, "response_length": 331, "context_tokens_target": 4000, "context_chars": 10108 }, { "iteration": 2, "ttft_ms": 165.67, "prefill_ms": 117.35, "decode_speed_tok_s": 79.5, "total_tokens": 5155, "prompt_tokens": 5075, "completion_tokens": 80, "e2e_ms": 1210.27, "elapsed_ms": 1210.27, "response_length": 326, "context_tokens_target": 4000, "context_chars": 10108 } ], "context_sizes_tested": [ 500, 1000, 2000, 4000 ] }, "statistics": { "count": 8, "mean_ms": 10733.07, "std_ms": 24421.91, "min_ms": 1109.55, "max_ms": 71112.83, "p50_ms": 1948.59, "p95_ms": 47714.2, "p99_ms": 66433.1 }, "inference_metrics": { "ttft_ms_mean": 9709.48, "ttft_ms_p95": 46669.91, "prefill_ms_mean": 935.8, "prefill_ms_p95": 2776.81, "decode_speed_tok_s_mean": 79.83, "decode_speed_tok_s_p95": 82.45, "e2e_ms_mean": 10733.07, "e2e_ms_p95": 47714.2, "total_tokens_mean": 2466.5, "total_tokens_p95": 5155.0 }, "elapsed_seconds": 85.87 }, { "case_id": "case_06_reasoning", "case_name": "复杂推理任务", "difficulty": "耗时", "status": "passed", "raw_data": { "case_id": "case_06_reasoning", "name": "复杂推理任务", "description": "不同推理复杂度下的延迟对比", "difficulty": "耗时", "estimated_seconds": 15, "results": [ { "iteration": 1, "ttft_ms": 502.6, "prefill_ms": 245.68, "decode_speed_tok_s": 80.34, "total_tokens": 93, "prompt_tokens": 13, "completion_tokens": 80, "e2e_ms": 1500.12, "elapsed_ms": 1500.12, "response_length": 320, "reasoning_level": "简单问答" }, { "iteration": 2, "ttft_ms": 220.67, "prefill_ms": 100.19, "decode_speed_tok_s": 83.2, "total_tokens": 93, "prompt_tokens": 13, "completion_tokens": 80, "e2e_ms": 1184.6, "elapsed_ms": 1184.6, "response_length": 317, "reasoning_level": "简单问答" }, { "iteration": 1, "ttft_ms": 438.71, "prefill_ms": 268.76, "decode_speed_tok_s": 82.97, "total_tokens": 220, "prompt_tokens": 20, "completion_tokens": 200, "e2e_ms": 2852.11, "elapsed_ms": 2852.11, "response_length": 714, "reasoning_level": "中等推理" }, { "iteration": 2, "ttft_ms": 232.27, "prefill_ms": 97.91, "decode_speed_tok_s": 84.27, "total_tokens": 220, "prompt_tokens": 20, "completion_tokens": 200, "e2e_ms": 2609.32, "elapsed_ms": 2609.32, "response_length": 861, "reasoning_level": "中等推理" }, { "iteration": 1, "ttft_ms": 471.77, "prefill_ms": 297.05, "decode_speed_tok_s": 83.04, "total_tokens": 427, "prompt_tokens": 27, "completion_tokens": 400, "e2e_ms": 5293.99, "elapsed_ms": 5293.99, "response_length": 1768, "reasoning_level": "复杂推理" }, { "iteration": 2, "ttft_ms": 234.98, "prefill_ms": 101.34, "decode_speed_tok_s": 82.71, "total_tokens": 427, "prompt_tokens": 27, "completion_tokens": 400, "e2e_ms": 5076.1, "elapsed_ms": 5076.1, "response_length": 1833, "reasoning_level": "复杂推理" } ] }, "statistics": { "count": 6, "mean_ms": 3086.04, "std_ms": 1746.31, "min_ms": 1184.6, "max_ms": 5293.99, "p50_ms": 2730.72, "p95_ms": 5239.52, "p99_ms": 5283.1 }, "inference_metrics": { "ttft_ms_mean": 350.17, "ttft_ms_p95": 494.89, "prefill_ms_mean": 185.16, "prefill_ms_p95": 289.98, "decode_speed_tok_s_mean": 82.75, "decode_speed_tok_s_p95": 84.0, "e2e_ms_mean": 3086.04, "e2e_ms_p95": 5239.52, "total_tokens_mean": 246.67, "total_tokens_p95": 427.0 }, "elapsed_seconds": 18.52 }, { "case_id": "case_07_multiturn", "case_name": "多轮对话累积", "difficulty": "耗时", "status": "failed", "error": "run_test() got an unexpected keyword argument 'repeats'", "elapsed_seconds": 0.0 } ] }