File size: 841 Bytes
29f7fc2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 |
{
"current_gpu_type": "NVIDIA L40S",
"current_gpu_total_memory": 45372.6875,
"memory_inference_first": 4312.0,
"memory_inference": 4208.0,
"token_generation_latency_sync": 47.82877426147461,
"token_generation_latency_async": 48.09337295591831,
"token_generation_throughput_sync": 0.020907916112027268,
"token_generation_throughput_async": 0.02079288555861086,
"token_generation_CO2_emissions": 3.164995601682604e-06,
"token_generation_energy_consumption": 0.00230724002830658,
"inference_latency_sync": 41.39498291015625,
"inference_latency_async": 40.357160568237305,
"inference_throughput_sync": 0.02415751691866625,
"inference_throughput_async": 0.024778750187569934,
"inference_CO2_emissions": 3.351062981628799e-06,
"inference_energy_consumption": 1.1458303099082523e-05
} |