1528 lines
44 KiB
JSON
1528 lines
44 KiB
JSON
{
|
|
"version": "1.0",
|
|
"timestamp": "2026-08-26T00:48:23.163258",
|
|
"run_id": "20260826-004823",
|
|
"environment": {
|
|
"python_version": "3.12.0",
|
|
"platform": "win32",
|
|
"model": "Q3.6-35B-A3B-Orig-Thi",
|
|
"backend": "openai"
|
|
},
|
|
"config": {
|
|
"cases_requested": [
|
|
"case_01_generation",
|
|
"case_08_concurrency"
|
|
],
|
|
"repeats": 1,
|
|
"skip_heavy": false,
|
|
"total_cases": 2
|
|
},
|
|
"summary": {
|
|
"total_cases": 2,
|
|
"passed": 2,
|
|
"failed": 0,
|
|
"stopped": false,
|
|
"total_elapsed_seconds": 60.01
|
|
},
|
|
"results": [
|
|
{
|
|
"case_id": "case_01_generation",
|
|
"case_name": "纯文本生成基准",
|
|
"difficulty": "简单",
|
|
"status": "passed",
|
|
"raw_data": {
|
|
"case_id": "case_01_generation",
|
|
"name": "纯文本生成基准",
|
|
"description": "基础生成速度基准测试",
|
|
"difficulty": "简单",
|
|
"estimated_seconds": 5,
|
|
"results": [
|
|
{
|
|
"iteration": 1,
|
|
"ttft_ms": 1361.65,
|
|
"prefill_ms": 174.46,
|
|
"decode_speed_tok_s": 55.94,
|
|
"total_tokens": 331,
|
|
"prompt_tokens": 31,
|
|
"completion_tokens": 300,
|
|
"e2e_ms": 5672.64,
|
|
"elapsed_ms": 5672.64,
|
|
"response_length": 917
|
|
}
|
|
],
|
|
"prompt_used": "请用200字左右概述人工智能从诞生至今的关键发展阶段，每段不超过50字。"
|
|
},
|
|
"statistics": {
|
|
"count": 1,
|
|
"mean_ms": 5672.64,
|
|
"std_ms": 0.0,
|
|
"min_ms": 5672.64,
|
|
"max_ms": 5672.64,
|
|
"p50_ms": 5672.64,
|
|
"p95_ms": 5672.64,
|
|
"p99_ms": 5672.64
|
|
},
|
|
"inference_metrics": {
|
|
"ttft_ms_mean": 1361.65,
|
|
"ttft_ms_p95": 1361.65,
|
|
"prefill_ms_mean": 174.46,
|
|
"prefill_ms_p95": 174.46,
|
|
"e2e_ms_mean": 5672.64,
|
|
"e2e_ms_p95": 5672.64,
|
|
"decode_speed_tok_s_mean": 55.94,
|
|
"decode_speed_tok_s_p95": 55.94,
|
|
"total_tokens_mean": 331.0,
|
|
"total_tokens_p95": 331
|
|
},
|
|
"elapsed_seconds": 5.67
|
|
},
|
|
{
|
|
"case_id": "case_08_concurrency",
|
|
"case_name": "并发压力测试",
|
|
"difficulty": "压力",
|
|
"status": "passed",
|
|
"raw_data": {
|
|
"case_id": "case_08_concurrency",
|
|
"name": "并发压力测试",
|
|
"description": "高并发下的 TPS 保持率、P95/P99 延迟、错误率",
|
|
"difficulty": "压力",
|
|
"estimated_seconds": 30,
|
|
"results": [
|
|
{
|
|
"iteration": 1,
|
|
"ttft_ms": 60.74,
|
|
"prefill_ms": 60.74,
|
|
"decode_speed_tok_s": 35.36,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 5674.36,
|
|
"elapsed_ms": 5674.36,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"ttft_ms": 56.05,
|
|
"prefill_ms": 56.05,
|
|
"decode_speed_tok_s": 41.47,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 5264.59,
|
|
"elapsed_ms": 5264.59,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 3,
|
|
"ttft_ms": 66.16,
|
|
"prefill_ms": 66.16,
|
|
"decode_speed_tok_s": 38.66,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 6600.18,
|
|
"elapsed_ms": 6600.18,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 4,
|
|
"ttft_ms": 68.09,
|
|
"prefill_ms": 68.09,
|
|
"decode_speed_tok_s": 38.55,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 6216.05,
|
|
"elapsed_ms": 6216.05,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 5,
|
|
"ttft_ms": 68.2,
|
|
"prefill_ms": 68.2,
|
|
"decode_speed_tok_s": 38.74,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 7129.09,
|
|
"elapsed_ms": 7129.09,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 6,
|
|
"ttft_ms": 66.17,
|
|
"prefill_ms": 66.17,
|
|
"decode_speed_tok_s": 38.73,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 7458.59,
|
|
"elapsed_ms": 7458.59,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 7,
|
|
"ttft_ms": 68.2,
|
|
"prefill_ms": 68.2,
|
|
"decode_speed_tok_s": 38.84,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 8022.2,
|
|
"elapsed_ms": 8022.2,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 8,
|
|
"ttft_ms": 66.1,
|
|
"prefill_ms": 66.1,
|
|
"decode_speed_tok_s": 38.64,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 8373.14,
|
|
"elapsed_ms": 8373.14,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 9,
|
|
"ttft_ms": 68.25,
|
|
"prefill_ms": 68.25,
|
|
"decode_speed_tok_s": 38.64,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 8911.84,
|
|
"elapsed_ms": 8911.84,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 10,
|
|
"ttft_ms": 66.69,
|
|
"prefill_ms": 66.69,
|
|
"decode_speed_tok_s": 38.6,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 9254.79,
|
|
"elapsed_ms": 9254.79,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 11,
|
|
"ttft_ms": 68.54,
|
|
"prefill_ms": 68.54,
|
|
"decode_speed_tok_s": 38.71,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 9801.75,
|
|
"elapsed_ms": 9801.75,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 12,
|
|
"ttft_ms": 66.39,
|
|
"prefill_ms": 66.39,
|
|
"decode_speed_tok_s": 38.66,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 10199.93,
|
|
"elapsed_ms": 10199.93,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 13,
|
|
"ttft_ms": 68.4,
|
|
"prefill_ms": 68.4,
|
|
"decode_speed_tok_s": 38.74,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 10687.59,
|
|
"elapsed_ms": 10687.59,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 14,
|
|
"ttft_ms": 66.35,
|
|
"prefill_ms": 66.35,
|
|
"decode_speed_tok_s": 38.71,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 11063.96,
|
|
"elapsed_ms": 11063.96,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 15,
|
|
"ttft_ms": 68.24,
|
|
"prefill_ms": 68.24,
|
|
"decode_speed_tok_s": 38.75,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 11593.13,
|
|
"elapsed_ms": 11593.13,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 16,
|
|
"ttft_ms": 66.28,
|
|
"prefill_ms": 66.28,
|
|
"decode_speed_tok_s": 38.68,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 11955.55,
|
|
"elapsed_ms": 11955.55,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 17,
|
|
"ttft_ms": 68.29,
|
|
"prefill_ms": 68.29,
|
|
"decode_speed_tok_s": 38.66,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 12490.11,
|
|
"elapsed_ms": 12490.11,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 18,
|
|
"ttft_ms": 66.25,
|
|
"prefill_ms": 66.25,
|
|
"decode_speed_tok_s": 38.48,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 12881.69,
|
|
"elapsed_ms": 12881.69,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 19,
|
|
"ttft_ms": 69.41,
|
|
"prefill_ms": 69.41,
|
|
"decode_speed_tok_s": 38.7,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 13384.74,
|
|
"elapsed_ms": 13384.74,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 20,
|
|
"ttft_ms": 66.24,
|
|
"prefill_ms": 66.24,
|
|
"decode_speed_tok_s": 38.74,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 13780.65,
|
|
"elapsed_ms": 13780.65,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 21,
|
|
"ttft_ms": 68.29,
|
|
"prefill_ms": 68.29,
|
|
"decode_speed_tok_s": 38.82,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 14265.17,
|
|
"elapsed_ms": 14265.17,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 22,
|
|
"ttft_ms": 66.26,
|
|
"prefill_ms": 66.26,
|
|
"decode_speed_tok_s": 38.75,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 14663.09,
|
|
"elapsed_ms": 14663.09,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 23,
|
|
"ttft_ms": 68.27,
|
|
"prefill_ms": 68.27,
|
|
"decode_speed_tok_s": 38.74,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 15171.04,
|
|
"elapsed_ms": 15171.04,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 24,
|
|
"ttft_ms": 66.21,
|
|
"prefill_ms": 66.21,
|
|
"decode_speed_tok_s": 38.59,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 15530.12,
|
|
"elapsed_ms": 15530.12,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 25,
|
|
"ttft_ms": 68.57,
|
|
"prefill_ms": 68.57,
|
|
"decode_speed_tok_s": 38.71,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 16066.75,
|
|
"elapsed_ms": 16066.75,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 26,
|
|
"ttft_ms": 66.25,
|
|
"prefill_ms": 66.25,
|
|
"decode_speed_tok_s": 38.69,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 16431.91,
|
|
"elapsed_ms": 16431.91,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 27,
|
|
"ttft_ms": 68.22,
|
|
"prefill_ms": 68.22,
|
|
"decode_speed_tok_s": 38.73,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 16970.02,
|
|
"elapsed_ms": 16970.02,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 28,
|
|
"ttft_ms": 66.24,
|
|
"prefill_ms": 66.24,
|
|
"decode_speed_tok_s": 38.68,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 17361.72,
|
|
"elapsed_ms": 17361.72,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 29,
|
|
"ttft_ms": 68.68,
|
|
"prefill_ms": 68.68,
|
|
"decode_speed_tok_s": 38.87,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 17844.61,
|
|
"elapsed_ms": 17844.61,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 30,
|
|
"ttft_ms": 66.18,
|
|
"prefill_ms": 66.18,
|
|
"decode_speed_tok_s": 38.78,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 18225.63,
|
|
"elapsed_ms": 18225.63,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 31,
|
|
"ttft_ms": 68.19,
|
|
"prefill_ms": 68.19,
|
|
"decode_speed_tok_s": 38.74,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 18774.54,
|
|
"elapsed_ms": 18774.54,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 32,
|
|
"ttft_ms": 66.18,
|
|
"prefill_ms": 66.18,
|
|
"decode_speed_tok_s": 38.73,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 19127.95,
|
|
"elapsed_ms": 19127.95,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 33,
|
|
"ttft_ms": 68.14,
|
|
"prefill_ms": 68.14,
|
|
"decode_speed_tok_s": 38.58,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 19670.18,
|
|
"elapsed_ms": 19670.18,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 34,
|
|
"ttft_ms": 66.95,
|
|
"prefill_ms": 66.95,
|
|
"decode_speed_tok_s": 38.53,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 19937.24,
|
|
"elapsed_ms": 19937.24,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 35,
|
|
"ttft_ms": 68.24,
|
|
"prefill_ms": 68.24,
|
|
"decode_speed_tok_s": 38.54,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 20555.94,
|
|
"elapsed_ms": 20555.94,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 36,
|
|
"ttft_ms": 66.44,
|
|
"prefill_ms": 66.44,
|
|
"decode_speed_tok_s": 38.62,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 20937.87,
|
|
"elapsed_ms": 20937.87,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 37,
|
|
"ttft_ms": 68.19,
|
|
"prefill_ms": 68.19,
|
|
"decode_speed_tok_s": 38.76,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 21435.19,
|
|
"elapsed_ms": 21435.19,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 38,
|
|
"ttft_ms": 66.34,
|
|
"prefill_ms": 66.34,
|
|
"decode_speed_tok_s": 38.68,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 21786.48,
|
|
"elapsed_ms": 21786.48,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 39,
|
|
"ttft_ms": 68.17,
|
|
"prefill_ms": 68.17,
|
|
"decode_speed_tok_s": 38.9,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 20920.76,
|
|
"elapsed_ms": 20920.76,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 40,
|
|
"ttft_ms": 66.17,
|
|
"prefill_ms": 66.17,
|
|
"decode_speed_tok_s": 38.88,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 22682.65,
|
|
"elapsed_ms": 22682.65,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 41,
|
|
"ttft_ms": 68.1,
|
|
"prefill_ms": 68.1,
|
|
"decode_speed_tok_s": 38.73,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 23215.5,
|
|
"elapsed_ms": 23215.5,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 42,
|
|
"ttft_ms": 66.24,
|
|
"prefill_ms": 66.24,
|
|
"decode_speed_tok_s": 38.59,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 21973.18,
|
|
"elapsed_ms": 21973.18,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 43,
|
|
"ttft_ms": 68.27,
|
|
"prefill_ms": 68.27,
|
|
"decode_speed_tok_s": 38.88,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 22938.81,
|
|
"elapsed_ms": 22938.81,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 44,
|
|
"ttft_ms": 66.11,
|
|
"prefill_ms": 66.11,
|
|
"decode_speed_tok_s": 38.78,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 24501.4,
|
|
"elapsed_ms": 24501.4,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 45,
|
|
"ttft_ms": 68.22,
|
|
"prefill_ms": 68.22,
|
|
"decode_speed_tok_s": 38.85,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 25002.77,
|
|
"elapsed_ms": 25002.77,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 46,
|
|
"ttft_ms": 66.24,
|
|
"prefill_ms": 66.24,
|
|
"decode_speed_tok_s": 38.85,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 22689.68,
|
|
"elapsed_ms": 22689.68,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 47,
|
|
"ttft_ms": 68.39,
|
|
"prefill_ms": 68.39,
|
|
"decode_speed_tok_s": 38.91,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 23393.4,
|
|
"elapsed_ms": 23393.4,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 48,
|
|
"ttft_ms": 66.18,
|
|
"prefill_ms": 66.18,
|
|
"decode_speed_tok_s": 38.85,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 23347.92,
|
|
"elapsed_ms": 23347.92,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 49,
|
|
"ttft_ms": 68.23,
|
|
"prefill_ms": 68.23,
|
|
"decode_speed_tok_s": 38.94,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 23569.98,
|
|
"elapsed_ms": 23569.98,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 50,
|
|
"ttft_ms": 66.13,
|
|
"prefill_ms": 66.13,
|
|
"decode_speed_tok_s": 38.87,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 24929.72,
|
|
"elapsed_ms": 24929.72,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 51,
|
|
"ttft_ms": 68.34,
|
|
"prefill_ms": 68.34,
|
|
"decode_speed_tok_s": 39.0,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 25878.51,
|
|
"elapsed_ms": 25878.51,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 52,
|
|
"ttft_ms": 66.11,
|
|
"prefill_ms": 66.11,
|
|
"decode_speed_tok_s": 38.86,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 26206.96,
|
|
"elapsed_ms": 26206.96,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 53,
|
|
"ttft_ms": 68.05,
|
|
"prefill_ms": 68.05,
|
|
"decode_speed_tok_s": 38.85,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 25135.12,
|
|
"elapsed_ms": 25135.12,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 54,
|
|
"ttft_ms": 66.89,
|
|
"prefill_ms": 66.89,
|
|
"decode_speed_tok_s": 38.93,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 25209.91,
|
|
"elapsed_ms": 25209.91,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 55,
|
|
"ttft_ms": 68.05,
|
|
"prefill_ms": 68.05,
|
|
"decode_speed_tok_s": 38.88,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 25457.93,
|
|
"elapsed_ms": 25457.93,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 56,
|
|
"ttft_ms": 66.14,
|
|
"prefill_ms": 66.14,
|
|
"decode_speed_tok_s": 38.8,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 25833.38,
|
|
"elapsed_ms": 25833.38,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 57,
|
|
"ttft_ms": 68.05,
|
|
"prefill_ms": 68.05,
|
|
"decode_speed_tok_s": 38.79,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 25070.91,
|
|
"elapsed_ms": 25070.91,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 58,
|
|
"ttft_ms": 66.22,
|
|
"prefill_ms": 66.22,
|
|
"decode_speed_tok_s": 38.69,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 25137.48,
|
|
"elapsed_ms": 25137.48,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 59,
|
|
"ttft_ms": 69.17,
|
|
"prefill_ms": 69.17,
|
|
"decode_speed_tok_s": 38.94,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 25730.69,
|
|
"elapsed_ms": 25730.69,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 60,
|
|
"ttft_ms": 66.08,
|
|
"prefill_ms": 66.08,
|
|
"decode_speed_tok_s": 38.88,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 25735.77,
|
|
"elapsed_ms": 25735.77,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 61,
|
|
"ttft_ms": 68.08,
|
|
"prefill_ms": 68.08,
|
|
"decode_speed_tok_s": 38.87,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 26371.2,
|
|
"elapsed_ms": 26371.2,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 62,
|
|
"ttft_ms": 66.16,
|
|
"prefill_ms": 66.16,
|
|
"decode_speed_tok_s": 38.79,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 26661.55,
|
|
"elapsed_ms": 26661.55,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 63,
|
|
"ttft_ms": 68.08,
|
|
"prefill_ms": 68.08,
|
|
"decode_speed_tok_s": 39.07,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 27349.81,
|
|
"elapsed_ms": 27349.81,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 64,
|
|
"ttft_ms": 66.06,
|
|
"prefill_ms": 66.06,
|
|
"decode_speed_tok_s": 38.94,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 27720.81,
|
|
"elapsed_ms": 27720.81,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 65,
|
|
"ttft_ms": 68.17,
|
|
"prefill_ms": 68.17,
|
|
"decode_speed_tok_s": 38.8,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 28072.64,
|
|
"elapsed_ms": 28072.64,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 66,
|
|
"ttft_ms": 66.53,
|
|
"prefill_ms": 66.53,
|
|
"decode_speed_tok_s": 38.67,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 28089.05,
|
|
"elapsed_ms": 28089.05,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 67,
|
|
"ttft_ms": 68.21,
|
|
"prefill_ms": 68.21,
|
|
"decode_speed_tok_s": 38.84,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 29111.18,
|
|
"elapsed_ms": 29111.18,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 68,
|
|
"ttft_ms": 66.26,
|
|
"prefill_ms": 66.26,
|
|
"decode_speed_tok_s": 38.89,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 29487.39,
|
|
"elapsed_ms": 29487.39,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 69,
|
|
"ttft_ms": 68.03,
|
|
"prefill_ms": 68.03,
|
|
"decode_speed_tok_s": 38.91,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 29608.49,
|
|
"elapsed_ms": 29608.49,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 70,
|
|
"ttft_ms": 66.09,
|
|
"prefill_ms": 66.09,
|
|
"decode_speed_tok_s": 38.85,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 30064.46,
|
|
"elapsed_ms": 30064.46,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 71,
|
|
"ttft_ms": 68.52,
|
|
"prefill_ms": 68.52,
|
|
"decode_speed_tok_s": 38.92,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 30537.86,
|
|
"elapsed_ms": 30537.86,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 72,
|
|
"ttft_ms": 66.55,
|
|
"prefill_ms": 66.55,
|
|
"decode_speed_tok_s": 38.83,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 30794.6,
|
|
"elapsed_ms": 30794.6,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 73,
|
|
"ttft_ms": 68.67,
|
|
"prefill_ms": 68.67,
|
|
"decode_speed_tok_s": 39.11,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 37482.39,
|
|
"elapsed_ms": 37482.39,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 74,
|
|
"ttft_ms": 65.99,
|
|
"prefill_ms": 65.99,
|
|
"decode_speed_tok_s": 39.02,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 37869.44,
|
|
"elapsed_ms": 37869.44,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 75,
|
|
"ttft_ms": 68.5,
|
|
"prefill_ms": 68.5,
|
|
"decode_speed_tok_s": 39.11,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 31796.22,
|
|
"elapsed_ms": 31796.22,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 76,
|
|
"ttft_ms": 66.16,
|
|
"prefill_ms": 66.16,
|
|
"decode_speed_tok_s": 39.08,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 38776.85,
|
|
"elapsed_ms": 38776.85,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 77,
|
|
"ttft_ms": 67.94,
|
|
"prefill_ms": 67.94,
|
|
"decode_speed_tok_s": 39.16,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 39278.77,
|
|
"elapsed_ms": 39278.77,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 78,
|
|
"ttft_ms": 66.11,
|
|
"prefill_ms": 66.11,
|
|
"decode_speed_tok_s": 39.11,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 39626.86,
|
|
"elapsed_ms": 39626.86,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 79,
|
|
"ttft_ms": 67.89,
|
|
"prefill_ms": 67.89,
|
|
"decode_speed_tok_s": 39.2,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 40151.1,
|
|
"elapsed_ms": 40151.1,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 80,
|
|
"ttft_ms": 66.11,
|
|
"prefill_ms": 66.11,
|
|
"decode_speed_tok_s": 39.07,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 40534.17,
|
|
"elapsed_ms": 40534.17,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 81,
|
|
"ttft_ms": 67.98,
|
|
"prefill_ms": 67.98,
|
|
"decode_speed_tok_s": 39.09,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 41029.96,
|
|
"elapsed_ms": 41029.96,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 82,
|
|
"ttft_ms": 66.09,
|
|
"prefill_ms": 66.09,
|
|
"decode_speed_tok_s": 39.08,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 41436.63,
|
|
"elapsed_ms": 41436.63,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 83,
|
|
"ttft_ms": 68.0,
|
|
"prefill_ms": 68.0,
|
|
"decode_speed_tok_s": 39.21,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 41670.92,
|
|
"elapsed_ms": 41670.92,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 84,
|
|
"ttft_ms": 66.05,
|
|
"prefill_ms": 66.05,
|
|
"decode_speed_tok_s": 39.16,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 42300.91,
|
|
"elapsed_ms": 42300.91,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 85,
|
|
"ttft_ms": 67.92,
|
|
"prefill_ms": 67.92,
|
|
"decode_speed_tok_s": 39.18,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 42285.07,
|
|
"elapsed_ms": 42285.07,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 86,
|
|
"ttft_ms": 66.05,
|
|
"prefill_ms": 66.05,
|
|
"decode_speed_tok_s": 38.99,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 42387.23,
|
|
"elapsed_ms": 42387.23,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 87,
|
|
"ttft_ms": 68.57,
|
|
"prefill_ms": 68.57,
|
|
"decode_speed_tok_s": 39.08,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 42706.42,
|
|
"elapsed_ms": 42706.42,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 88,
|
|
"ttft_ms": 66.11,
|
|
"prefill_ms": 66.11,
|
|
"decode_speed_tok_s": 38.96,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 44074.46,
|
|
"elapsed_ms": 44074.46,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 89,
|
|
"ttft_ms": 67.82,
|
|
"prefill_ms": 67.82,
|
|
"decode_speed_tok_s": 39.06,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 44576.86,
|
|
"elapsed_ms": 44576.86,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 90,
|
|
"ttft_ms": 66.05,
|
|
"prefill_ms": 66.05,
|
|
"decode_speed_tok_s": 39.07,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 44959.12,
|
|
"elapsed_ms": 44959.12,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 91,
|
|
"ttft_ms": 68.2,
|
|
"prefill_ms": 68.2,
|
|
"decode_speed_tok_s": 39.25,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 44887.1,
|
|
"elapsed_ms": 44887.1,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 92,
|
|
"ttft_ms": 65.99,
|
|
"prefill_ms": 65.99,
|
|
"decode_speed_tok_s": 39.27,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 45825.94,
|
|
"elapsed_ms": 45825.94,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 93,
|
|
"ttft_ms": 67.79,
|
|
"prefill_ms": 67.79,
|
|
"decode_speed_tok_s": 39.23,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 45953.13,
|
|
"elapsed_ms": 45953.13,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 94,
|
|
"ttft_ms": 66.02,
|
|
"prefill_ms": 66.02,
|
|
"decode_speed_tok_s": 39.17,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 41492.84,
|
|
"elapsed_ms": 41492.84,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 95,
|
|
"ttft_ms": 67.86,
|
|
"prefill_ms": 67.86,
|
|
"decode_speed_tok_s": 39.32,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 42895.55,
|
|
"elapsed_ms": 42895.55,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 96,
|
|
"ttft_ms": 65.97,
|
|
"prefill_ms": 65.97,
|
|
"decode_speed_tok_s": 39.26,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 42080.43,
|
|
"elapsed_ms": 42080.43,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 97,
|
|
"ttft_ms": 67.88,
|
|
"prefill_ms": 67.88,
|
|
"decode_speed_tok_s": 39.39,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 42992.32,
|
|
"elapsed_ms": 42992.32,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 98,
|
|
"ttft_ms": 65.96,
|
|
"prefill_ms": 65.96,
|
|
"decode_speed_tok_s": 39.28,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 43592.29,
|
|
"elapsed_ms": 43592.29,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 99,
|
|
"ttft_ms": 67.75,
|
|
"prefill_ms": 67.75,
|
|
"decode_speed_tok_s": 39.2,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 42537.22,
|
|
"elapsed_ms": 42537.22,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
},
|
|
{
|
|
"iteration": 100,
|
|
"ttft_ms": 68.47,
|
|
"prefill_ms": 68.47,
|
|
"decode_speed_tok_s": 46.89,
|
|
"total_tokens": 36,
|
|
"prompt_tokens": 4,
|
|
"completion_tokens": 32,
|
|
"e2e_ms": 44598.8,
|
|
"elapsed_ms": 44598.8,
|
|
"response_length": 91,
|
|
"batch": "concurrent",
|
|
"error": ""
|
|
}
|
|
],
|
|
"concurrency": 100,
|
|
"baseline_tps": 57.43,
|
|
"concurrent_tps": 38.95,
|
|
"tps_keep_rate_pct": 67.82,
|
|
"e2e_p50_ms": 25036.84,
|
|
"e2e_p95_ms": 44577.95,
|
|
"e2e_p99_ms": 45827.21,
|
|
"success_count": 100,
|
|
"failed_count": 0,
|
|
"error_rate_pct": 0.0
|
|
},
|
|
"statistics": {
|
|
"count": 100,
|
|
"mean_ms": 25637.3,
|
|
"std_ms": 11979.73,
|
|
"min_ms": 5264.59,
|
|
"max_ms": 45953.13,
|
|
"p50_ms": 25036.84,
|
|
"p95_ms": 44577.96,
|
|
"p99_ms": 45827.21
|
|
},
|
|
"inference_metrics": {
|
|
"ttft_ms_mean": 67.07,
|
|
"ttft_ms_p95": 68.57,
|
|
"prefill_ms_mean": 67.07,
|
|
"prefill_ms_p95": 68.57,
|
|
"e2e_ms_mean": 25637.3,
|
|
"e2e_ms_p95": 44577.96,
|
|
"decode_speed_tok_s_mean": 38.95,
|
|
"decode_speed_tok_s_p95": 39.27,
|
|
"total_tokens_mean": 36.0,
|
|
"total_tokens_p95": 36.0
|
|
},
|
|
"elapsed_seconds": 54.33
|
|
}
|
|
]
|
|
} |