{ "version": "1.0", "timestamp": "2026-08-26T00:48:23.163258", "run_id": "20260826-004823", "environment": { "python_version": "3.12.0", "platform": "win32", "model": "Q3.6-35B-A3B-Orig-Thi", "backend": "openai" }, "config": { "cases_requested": [ "case_01_generation", "case_08_concurrency" ], "repeats": 1, "skip_heavy": false, "total_cases": 2 }, "summary": { "total_cases": 2, "passed": 2, "failed": 0, "stopped": false, "total_elapsed_seconds": 60.01 }, "results": [ { "case_id": "case_01_generation", "case_name": "纯文本生成基准", "difficulty": "简单", "status": "passed", "raw_data": { "case_id": "case_01_generation", "name": "纯文本生成基准", "description": "基础生成速度基准测试", "difficulty": "简单", "estimated_seconds": 5, "results": [ { "iteration": 1, "ttft_ms": 1361.65, "prefill_ms": 174.46, "decode_speed_tok_s": 55.94, "total_tokens": 331, "prompt_tokens": 31, "completion_tokens": 300, "e2e_ms": 5672.64, "elapsed_ms": 5672.64, "response_length": 917 } ], "prompt_used": "请用200字左右概述人工智能从诞生至今的关键发展阶段，每段不超过50字。" }, "statistics": { "count": 1, "mean_ms": 5672.64, "std_ms": 0.0, "min_ms": 5672.64, "max_ms": 5672.64, "p50_ms": 5672.64, "p95_ms": 5672.64, "p99_ms": 5672.64 }, "inference_metrics": { "ttft_ms_mean": 1361.65, "ttft_ms_p95": 1361.65, "prefill_ms_mean": 174.46, "prefill_ms_p95": 174.46, "e2e_ms_mean": 5672.64, "e2e_ms_p95": 5672.64, "decode_speed_tok_s_mean": 55.94, "decode_speed_tok_s_p95": 55.94, "total_tokens_mean": 331.0, "total_tokens_p95": 331 }, "elapsed_seconds": 5.67 }, { "case_id": "case_08_concurrency", "case_name": "并发压力测试", "difficulty": "压力", "status": "passed", "raw_data": { "case_id": "case_08_concurrency", "name": "并发压力测试", "description": "高并发下的 TPS 保持率、P95/P99 延迟、错误率", "difficulty": "压力", "estimated_seconds": 30, "results": [ { "iteration": 1, "ttft_ms": 60.74, "prefill_ms": 60.74, "decode_speed_tok_s": 35.36, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 5674.36, "elapsed_ms": 5674.36, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 2, "ttft_ms": 56.05, "prefill_ms": 56.05, "decode_speed_tok_s": 41.47, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 5264.59, "elapsed_ms": 5264.59, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 3, "ttft_ms": 66.16, "prefill_ms": 66.16, "decode_speed_tok_s": 38.66, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 6600.18, "elapsed_ms": 6600.18, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 4, "ttft_ms": 68.09, "prefill_ms": 68.09, "decode_speed_tok_s": 38.55, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 6216.05, "elapsed_ms": 6216.05, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 5, "ttft_ms": 68.2, "prefill_ms": 68.2, "decode_speed_tok_s": 38.74, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 7129.09, "elapsed_ms": 7129.09, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 6, "ttft_ms": 66.17, "prefill_ms": 66.17, "decode_speed_tok_s": 38.73, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 7458.59, "elapsed_ms": 7458.59, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 7, "ttft_ms": 68.2, "prefill_ms": 68.2, "decode_speed_tok_s": 38.84, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 8022.2, "elapsed_ms": 8022.2, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 8, "ttft_ms": 66.1, "prefill_ms": 66.1, "decode_speed_tok_s": 38.64, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 8373.14, "elapsed_ms": 8373.14, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 9, "ttft_ms": 68.25, "prefill_ms": 68.25, "decode_speed_tok_s": 38.64, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 8911.84, "elapsed_ms": 8911.84, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 10, "ttft_ms": 66.69, "prefill_ms": 66.69, "decode_speed_tok_s": 38.6, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 9254.79, "elapsed_ms": 9254.79, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 11, "ttft_ms": 68.54, "prefill_ms": 68.54, "decode_speed_tok_s": 38.71, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 9801.75, "elapsed_ms": 9801.75, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 12, "ttft_ms": 66.39, "prefill_ms": 66.39, "decode_speed_tok_s": 38.66, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 10199.93, "elapsed_ms": 10199.93, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 13, "ttft_ms": 68.4, "prefill_ms": 68.4, "decode_speed_tok_s": 38.74, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 10687.59, "elapsed_ms": 10687.59, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 14, "ttft_ms": 66.35, "prefill_ms": 66.35, "decode_speed_tok_s": 38.71, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 11063.96, "elapsed_ms": 11063.96, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 15, "ttft_ms": 68.24, "prefill_ms": 68.24, "decode_speed_tok_s": 38.75, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 11593.13, "elapsed_ms": 11593.13, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 16, "ttft_ms": 66.28, "prefill_ms": 66.28, "decode_speed_tok_s": 38.68, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 11955.55, "elapsed_ms": 11955.55, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 17, "ttft_ms": 68.29, "prefill_ms": 68.29, "decode_speed_tok_s": 38.66, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 12490.11, "elapsed_ms": 12490.11, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 18, "ttft_ms": 66.25, "prefill_ms": 66.25, "decode_speed_tok_s": 38.48, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 12881.69, "elapsed_ms": 12881.69, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 19, "ttft_ms": 69.41, "prefill_ms": 69.41, "decode_speed_tok_s": 38.7, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 13384.74, "elapsed_ms": 13384.74, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 20, "ttft_ms": 66.24, "prefill_ms": 66.24, "decode_speed_tok_s": 38.74, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 13780.65, "elapsed_ms": 13780.65, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 21, "ttft_ms": 68.29, "prefill_ms": 68.29, "decode_speed_tok_s": 38.82, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 14265.17, "elapsed_ms": 14265.17, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 22, "ttft_ms": 66.26, "prefill_ms": 66.26, "decode_speed_tok_s": 38.75, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 14663.09, "elapsed_ms": 14663.09, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 23, "ttft_ms": 68.27, "prefill_ms": 68.27, "decode_speed_tok_s": 38.74, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 15171.04, "elapsed_ms": 15171.04, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 24, "ttft_ms": 66.21, "prefill_ms": 66.21, "decode_speed_tok_s": 38.59, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 15530.12, "elapsed_ms": 15530.12, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 25, "ttft_ms": 68.57, "prefill_ms": 68.57, "decode_speed_tok_s": 38.71, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 16066.75, "elapsed_ms": 16066.75, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 26, "ttft_ms": 66.25, "prefill_ms": 66.25, "decode_speed_tok_s": 38.69, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 16431.91, "elapsed_ms": 16431.91, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 27, "ttft_ms": 68.22, "prefill_ms": 68.22, "decode_speed_tok_s": 38.73, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 16970.02, "elapsed_ms": 16970.02, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 28, "ttft_ms": 66.24, "prefill_ms": 66.24, "decode_speed_tok_s": 38.68, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 17361.72, "elapsed_ms": 17361.72, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 29, "ttft_ms": 68.68, "prefill_ms": 68.68, "decode_speed_tok_s": 38.87, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 17844.61, "elapsed_ms": 17844.61, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 30, "ttft_ms": 66.18, "prefill_ms": 66.18, "decode_speed_tok_s": 38.78, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 18225.63, "elapsed_ms": 18225.63, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 31, "ttft_ms": 68.19, "prefill_ms": 68.19, "decode_speed_tok_s": 38.74, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 18774.54, "elapsed_ms": 18774.54, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 32, "ttft_ms": 66.18, "prefill_ms": 66.18, "decode_speed_tok_s": 38.73, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 19127.95, "elapsed_ms": 19127.95, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 33, "ttft_ms": 68.14, "prefill_ms": 68.14, "decode_speed_tok_s": 38.58, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 19670.18, "elapsed_ms": 19670.18, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 34, "ttft_ms": 66.95, "prefill_ms": 66.95, "decode_speed_tok_s": 38.53, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 19937.24, "elapsed_ms": 19937.24, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 35, "ttft_ms": 68.24, "prefill_ms": 68.24, "decode_speed_tok_s": 38.54, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 20555.94, "elapsed_ms": 20555.94, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 36, "ttft_ms": 66.44, "prefill_ms": 66.44, "decode_speed_tok_s": 38.62, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 20937.87, "elapsed_ms": 20937.87, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 37, "ttft_ms": 68.19, "prefill_ms": 68.19, "decode_speed_tok_s": 38.76, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 21435.19, "elapsed_ms": 21435.19, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 38, "ttft_ms": 66.34, "prefill_ms": 66.34, "decode_speed_tok_s": 38.68, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 21786.48, "elapsed_ms": 21786.48, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 39, "ttft_ms": 68.17, "prefill_ms": 68.17, "decode_speed_tok_s": 38.9, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 20920.76, "elapsed_ms": 20920.76, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 40, "ttft_ms": 66.17, "prefill_ms": 66.17, "decode_speed_tok_s": 38.88, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 22682.65, "elapsed_ms": 22682.65, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 41, "ttft_ms": 68.1, "prefill_ms": 68.1, "decode_speed_tok_s": 38.73, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 23215.5, "elapsed_ms": 23215.5, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 42, "ttft_ms": 66.24, "prefill_ms": 66.24, "decode_speed_tok_s": 38.59, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 21973.18, "elapsed_ms": 21973.18, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 43, "ttft_ms": 68.27, "prefill_ms": 68.27, "decode_speed_tok_s": 38.88, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 22938.81, "elapsed_ms": 22938.81, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 44, "ttft_ms": 66.11, "prefill_ms": 66.11, "decode_speed_tok_s": 38.78, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 24501.4, "elapsed_ms": 24501.4, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 45, "ttft_ms": 68.22, "prefill_ms": 68.22, "decode_speed_tok_s": 38.85, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 25002.77, "elapsed_ms": 25002.77, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 46, "ttft_ms": 66.24, "prefill_ms": 66.24, "decode_speed_tok_s": 38.85, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 22689.68, "elapsed_ms": 22689.68, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 47, "ttft_ms": 68.39, "prefill_ms": 68.39, "decode_speed_tok_s": 38.91, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 23393.4, "elapsed_ms": 23393.4, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 48, "ttft_ms": 66.18, "prefill_ms": 66.18, "decode_speed_tok_s": 38.85, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 23347.92, "elapsed_ms": 23347.92, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 49, "ttft_ms": 68.23, "prefill_ms": 68.23, "decode_speed_tok_s": 38.94, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 23569.98, "elapsed_ms": 23569.98, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 50, "ttft_ms": 66.13, "prefill_ms": 66.13, "decode_speed_tok_s": 38.87, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 24929.72, "elapsed_ms": 24929.72, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 51, "ttft_ms": 68.34, "prefill_ms": 68.34, "decode_speed_tok_s": 39.0, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 25878.51, "elapsed_ms": 25878.51, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 52, "ttft_ms": 66.11, "prefill_ms": 66.11, "decode_speed_tok_s": 38.86, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 26206.96, "elapsed_ms": 26206.96, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 53, "ttft_ms": 68.05, "prefill_ms": 68.05, "decode_speed_tok_s": 38.85, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 25135.12, "elapsed_ms": 25135.12, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 54, "ttft_ms": 66.89, "prefill_ms": 66.89, "decode_speed_tok_s": 38.93, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 25209.91, "elapsed_ms": 25209.91, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 55, "ttft_ms": 68.05, "prefill_ms": 68.05, "decode_speed_tok_s": 38.88, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 25457.93, "elapsed_ms": 25457.93, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 56, "ttft_ms": 66.14, "prefill_ms": 66.14, "decode_speed_tok_s": 38.8, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 25833.38, "elapsed_ms": 25833.38, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 57, "ttft_ms": 68.05, "prefill_ms": 68.05, "decode_speed_tok_s": 38.79, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 25070.91, "elapsed_ms": 25070.91, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 58, "ttft_ms": 66.22, "prefill_ms": 66.22, "decode_speed_tok_s": 38.69, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 25137.48, "elapsed_ms": 25137.48, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 59, "ttft_ms": 69.17, "prefill_ms": 69.17, "decode_speed_tok_s": 38.94, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 25730.69, "elapsed_ms": 25730.69, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 60, "ttft_ms": 66.08, "prefill_ms": 66.08, "decode_speed_tok_s": 38.88, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 25735.77, "elapsed_ms": 25735.77, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 61, "ttft_ms": 68.08, "prefill_ms": 68.08, "decode_speed_tok_s": 38.87, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 26371.2, "elapsed_ms": 26371.2, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 62, "ttft_ms": 66.16, "prefill_ms": 66.16, "decode_speed_tok_s": 38.79, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 26661.55, "elapsed_ms": 26661.55, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 63, "ttft_ms": 68.08, "prefill_ms": 68.08, "decode_speed_tok_s": 39.07, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 27349.81, "elapsed_ms": 27349.81, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 64, "ttft_ms": 66.06, "prefill_ms": 66.06, "decode_speed_tok_s": 38.94, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 27720.81, "elapsed_ms": 27720.81, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 65, "ttft_ms": 68.17, "prefill_ms": 68.17, "decode_speed_tok_s": 38.8, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 28072.64, "elapsed_ms": 28072.64, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 66, "ttft_ms": 66.53, "prefill_ms": 66.53, "decode_speed_tok_s": 38.67, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 28089.05, "elapsed_ms": 28089.05, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 67, "ttft_ms": 68.21, "prefill_ms": 68.21, "decode_speed_tok_s": 38.84, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 29111.18, "elapsed_ms": 29111.18, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 68, "ttft_ms": 66.26, "prefill_ms": 66.26, "decode_speed_tok_s": 38.89, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 29487.39, "elapsed_ms": 29487.39, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 69, "ttft_ms": 68.03, "prefill_ms": 68.03, "decode_speed_tok_s": 38.91, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 29608.49, "elapsed_ms": 29608.49, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 70, "ttft_ms": 66.09, "prefill_ms": 66.09, "decode_speed_tok_s": 38.85, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 30064.46, "elapsed_ms": 30064.46, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 71, "ttft_ms": 68.52, "prefill_ms": 68.52, "decode_speed_tok_s": 38.92, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 30537.86, "elapsed_ms": 30537.86, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 72, "ttft_ms": 66.55, "prefill_ms": 66.55, "decode_speed_tok_s": 38.83, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 30794.6, "elapsed_ms": 30794.6, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 73, "ttft_ms": 68.67, "prefill_ms": 68.67, "decode_speed_tok_s": 39.11, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 37482.39, "elapsed_ms": 37482.39, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 74, "ttft_ms": 65.99, "prefill_ms": 65.99, "decode_speed_tok_s": 39.02, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 37869.44, "elapsed_ms": 37869.44, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 75, "ttft_ms": 68.5, "prefill_ms": 68.5, "decode_speed_tok_s": 39.11, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 31796.22, "elapsed_ms": 31796.22, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 76, "ttft_ms": 66.16, "prefill_ms": 66.16, "decode_speed_tok_s": 39.08, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 38776.85, "elapsed_ms": 38776.85, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 77, "ttft_ms": 67.94, "prefill_ms": 67.94, "decode_speed_tok_s": 39.16, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 39278.77, "elapsed_ms": 39278.77, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 78, "ttft_ms": 66.11, "prefill_ms": 66.11, "decode_speed_tok_s": 39.11, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 39626.86, "elapsed_ms": 39626.86, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 79, "ttft_ms": 67.89, "prefill_ms": 67.89, "decode_speed_tok_s": 39.2, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 40151.1, "elapsed_ms": 40151.1, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 80, "ttft_ms": 66.11, "prefill_ms": 66.11, "decode_speed_tok_s": 39.07, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 40534.17, "elapsed_ms": 40534.17, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 81, "ttft_ms": 67.98, "prefill_ms": 67.98, "decode_speed_tok_s": 39.09, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 41029.96, "elapsed_ms": 41029.96, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 82, "ttft_ms": 66.09, "prefill_ms": 66.09, "decode_speed_tok_s": 39.08, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 41436.63, "elapsed_ms": 41436.63, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 83, "ttft_ms": 68.0, "prefill_ms": 68.0, "decode_speed_tok_s": 39.21, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 41670.92, "elapsed_ms": 41670.92, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 84, "ttft_ms": 66.05, "prefill_ms": 66.05, "decode_speed_tok_s": 39.16, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 42300.91, "elapsed_ms": 42300.91, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 85, "ttft_ms": 67.92, "prefill_ms": 67.92, "decode_speed_tok_s": 39.18, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 42285.07, "elapsed_ms": 42285.07, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 86, "ttft_ms": 66.05, "prefill_ms": 66.05, "decode_speed_tok_s": 38.99, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 42387.23, "elapsed_ms": 42387.23, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 87, "ttft_ms": 68.57, "prefill_ms": 68.57, "decode_speed_tok_s": 39.08, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 42706.42, "elapsed_ms": 42706.42, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 88, "ttft_ms": 66.11, "prefill_ms": 66.11, "decode_speed_tok_s": 38.96, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 44074.46, "elapsed_ms": 44074.46, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 89, "ttft_ms": 67.82, "prefill_ms": 67.82, "decode_speed_tok_s": 39.06, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 44576.86, "elapsed_ms": 44576.86, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 90, "ttft_ms": 66.05, "prefill_ms": 66.05, "decode_speed_tok_s": 39.07, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 44959.12, "elapsed_ms": 44959.12, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 91, "ttft_ms": 68.2, "prefill_ms": 68.2, "decode_speed_tok_s": 39.25, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 44887.1, "elapsed_ms": 44887.1, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 92, "ttft_ms": 65.99, "prefill_ms": 65.99, "decode_speed_tok_s": 39.27, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 45825.94, "elapsed_ms": 45825.94, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 93, "ttft_ms": 67.79, "prefill_ms": 67.79, "decode_speed_tok_s": 39.23, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 45953.13, "elapsed_ms": 45953.13, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 94, "ttft_ms": 66.02, "prefill_ms": 66.02, "decode_speed_tok_s": 39.17, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 41492.84, "elapsed_ms": 41492.84, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 95, "ttft_ms": 67.86, "prefill_ms": 67.86, "decode_speed_tok_s": 39.32, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 42895.55, "elapsed_ms": 42895.55, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 96, "ttft_ms": 65.97, "prefill_ms": 65.97, "decode_speed_tok_s": 39.26, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 42080.43, "elapsed_ms": 42080.43, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 97, "ttft_ms": 67.88, "prefill_ms": 67.88, "decode_speed_tok_s": 39.39, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 42992.32, "elapsed_ms": 42992.32, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 98, "ttft_ms": 65.96, "prefill_ms": 65.96, "decode_speed_tok_s": 39.28, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 43592.29, "elapsed_ms": 43592.29, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 99, "ttft_ms": 67.75, "prefill_ms": 67.75, "decode_speed_tok_s": 39.2, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 42537.22, "elapsed_ms": 42537.22, "response_length": 91, "batch": "concurrent", "error": "" }, { "iteration": 100, "ttft_ms": 68.47, "prefill_ms": 68.47, "decode_speed_tok_s": 46.89, "total_tokens": 36, "prompt_tokens": 4, "completion_tokens": 32, "e2e_ms": 44598.8, "elapsed_ms": 44598.8, "response_length": 91, "batch": "concurrent", "error": "" } ], "concurrency": 100, "baseline_tps": 57.43, "concurrent_tps": 38.95, "tps_keep_rate_pct": 67.82, "e2e_p50_ms": 25036.84, "e2e_p95_ms": 44577.95, "e2e_p99_ms": 45827.21, "success_count": 100, "failed_count": 0, "error_rate_pct": 0.0 }, "statistics": { "count": 100, "mean_ms": 25637.3, "std_ms": 11979.73, "min_ms": 5264.59, "max_ms": 45953.13, "p50_ms": 25036.84, "p95_ms": 44577.96, "p99_ms": 45827.21 }, "inference_metrics": { "ttft_ms_mean": 67.07, "ttft_ms_p95": 68.57, "prefill_ms_mean": 67.07, "prefill_ms_p95": 68.57, "e2e_ms_mean": 25637.3, "e2e_ms_p95": 44577.96, "decode_speed_tok_s_mean": 38.95, "decode_speed_tok_s_p95": 39.27, "total_tokens_mean": 36.0, "total_tokens_p95": 36.0 }, "elapsed_seconds": 54.33 } ] }