Upload /gemma-2b/int8_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

gemma-2b/int8_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,552 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1423.818752,
+            "max_vram": 4257.21856,
+            "max_reserved": 3764.38784,
+            "max_allocated": 3508.004352
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.04286323381081605,
+            "stdev": 0.000656232047182085,
+            "values": [
+                0.0505134391784668,
+                0.04533657455444336,
+                0.04497100830078125,
+                0.044565502166748046,
+                0.04431257629394531,
+                0.04437094497680664,
+                0.04474265670776367,
+                0.04444384002685547,
+                0.0444815673828125,
+                0.04461772918701172,
+                0.04460236740112305,
+                0.043878398895263675,
+                0.04398489761352539,
+                0.04345446395874023,
+                0.04287078475952148,
+                0.042608638763427735,
+                0.0425615348815918,
+                0.04250009536743164,
+                0.04266393661499023,
+                0.04250431823730469,
+                0.04270182418823242,
+                0.04260659027099609,
+                0.042608638763427735,
+                0.04252569580078125,
+                0.04266393661499023,
+                0.04261068725585938,
+                0.042559486389160156,
+                0.04261171340942383,
+                0.04253286361694336,
+                0.04257894515991211,
+                0.04250009536743164,
+                0.04264038467407227,
+                0.04261785507202148,
+                0.04275404739379883,
+                0.042654720306396485,
+                0.04260464096069336,
+                0.0425984001159668,
+                0.042649600982666014,
+                0.04270182418823242,
+                0.04271104049682617,
+                0.042659839630126956,
+                0.042799102783203126,
+                0.04271615982055664,
+                0.042560672760009764,
+                0.04260454559326172,
+                0.04265267181396484,
+                0.04253696060180664,
+                0.0426956787109375,
+                0.04271206283569336,
+                0.04274585723876953,
+                0.04262809753417969,
+                0.04266291046142578,
+                0.042589183807373046,
+                0.042613761901855465,
+                0.042587135314941404,
+                0.04259651184082031,
+                0.04278579330444336,
+                0.042850303649902347,
+                0.04276428985595703,
+                0.04276326370239258,
+                0.042676223754882815,
+                0.04270489501953125,
+                0.04269875335693359,
+                0.042891265869140625,
+                0.0427509765625,
+                0.04275711822509766,
+                0.042889217376708984,
+                0.04270182418823242,
+                0.04268134307861328,
+                0.042646526336669925,
+                0.042782718658447266,
+                0.04276838302612305,
+                0.04274176025390625,
+                0.04274790573120117,
+                0.042726398468017575,
+                0.042831871032714845,
+                0.042823680877685545,
+                0.04273561477661133,
+                0.04283087921142578,
+                0.04281753540039063,
+                0.0427470703125,
+                0.04268032073974609,
+                0.04280627059936523,
+                0.04269363021850586,
+                0.04283391952514649,
+                0.04278374481201172,
+                0.04267724609375,
+                0.042777599334716795,
+                0.04269465637207031,
+                0.042671104431152344,
+                0.04271308898925781,
+                0.04274687957763672,
+                0.042641407012939454,
+                0.04270694351196289,
+                0.042726398468017575,
+                0.042821632385253904,
+                0.042673152923583986,
+                0.04267007827758789,
+                0.04262911987304688,
+                0.04268851089477539,
+                0.04268646240234375,
+                0.04269055938720703,
+                0.04262604904174805,
+                0.04266495895385742,
+                0.04266291046142578,
+                0.042643455505371096,
+                0.04258816146850586,
+                0.042602497100830077,
+                0.042722305297851565,
+                0.04276736068725586,
+                0.04278988647460937,
+                0.04261395263671875,
+                0.04253593444824219,
+                0.042517505645751956,
+                0.042742942810058596,
+                0.04272435379028321,
+                0.04267520141601563,
+                0.042771457672119144,
+                0.04271718215942383,
+                0.042651649475097655,
+                0.04262604904174805,
+                0.042687488555908204,
+                0.04273152160644531,
+                0.04275302505493164,
+                0.04270182418823242,
+                0.04261580657958984,
+                0.04272435379028321,
+                0.04262297439575195,
+                0.042665985107421874,
+                0.04257894515991211,
+                0.042703872680664064,
+                0.04265574264526367,
+                0.04274585723876953,
+                0.04276838302612305,
+                0.04268236923217773,
+                0.04266495895385742,
+                0.04276428985595703,
+                0.042643455505371096,
+                0.04256256103515625,
+                0.04258508682250976,
+                0.042807296752929686,
+                0.042656768798828126,
+                0.042638336181640625,
+                0.042554367065429685,
+                0.04266291046142578,
+                0.042635265350341796,
+                0.042600448608398435,
+                0.04242227172851563,
+                0.0425596809387207,
+                0.042420223236083986,
+                0.042641407012939454,
+                0.04266291046142578,
+                0.04285747146606445,
+                0.04269772720336914,
+                0.04276428985595703,
+                0.04269363021850586,
+                0.042543102264404296,
+                0.04262604904174805,
+                0.04257689666748047,
+                0.042728446960449216,
+                0.04267212677001953,
+                0.0427509765625,
+                0.042692607879638675,
+                0.042584064483642575,
+                0.042845184326171876,
+                0.042829822540283204,
+                0.0427694091796875,
+                0.04266499328613281,
+                0.042777599334716795,
+                0.042840065002441405,
+                0.04283494567871094,
+                0.04268032073974609,
+                0.042700801849365234,
+                0.042828800201416016,
+                0.0428328971862793,
+                0.04269158554077149,
+                0.042788864135742184,
+                0.042858497619628906,
+                0.04276326370239258,
+                0.04282572937011719,
+                0.042943489074707034,
+                0.04280115127563477,
+                0.04270182418823242,
+                0.04277657699584961,
+                0.04269977569580078,
+                0.04282275390625,
+                0.04274790573120117,
+                0.04281651306152344,
+                0.042916862487792966,
+                0.04290969467163086,
+                0.04278579330444336,
+                0.04275814437866211,
+                0.04273664093017578,
+                0.042856449127197264,
+                0.04293030548095703,
+                0.042823680877685545,
+                0.04281651306152344,
+                0.04284620666503906,
+                0.04276649475097656,
+                0.042840065002441405,
+                0.04283391952514649,
+                0.04290867233276367,
+                0.04290867233276367,
+                0.042859519958496094,
+                0.042874881744384766,
+                0.042777599334716795,
+                0.042984447479248046,
+                0.04295782470703125,
+                0.04300201416015625,
+                0.042851329803466794,
+                0.04295577621459961,
+                0.042928192138671876,
+                0.042900478363037106,
+                0.04290560150146484,
+                0.04297011184692383,
+                0.04281651306152344,
+                0.042831871032714845,
+                0.04276428985595703,
+                0.04290560150146484,
+                0.042799102783203126,
+                0.04309401702880859,
+                0.04304076766967774,
+                0.04305100631713867,
+                0.042947582244873043,
+                0.04299980926513672,
+                0.04299673461914062,
+                0.04303974533081055,
+                0.0429639663696289,
+                0.042927104949951174,
+                0.04294041442871094,
+                0.042927104949951174,
+                0.04289945602416992,
+                0.042881023406982424,
+                0.04286259078979492
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 5972.484510382446
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1424.867328,
+            "max_vram": 4534.042624,
+            "max_reserved": 4041.211904,
+            "max_allocated": 3911.199744
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 11.125907440185541,
+            "stdev": 0,
+            "values": [
+                11.125907440185541
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 22.91947882641629
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.04363100956935506,
+            "stdev": 0.00011434622885004557,
+            "values": [
+                0.04358553695678711,
+                0.04352716827392578,
+                0.04343807983398437,
+                0.043655166625976564,
+                0.043483135223388675,
+                0.04352511978149414,
+                0.04334592056274414,
+                0.04340326309204102,
+                0.04347187042236328,
+                0.04380057525634766,
+                0.04369715118408203,
+                0.04368896102905274,
+                0.043619327545166016,
+                0.04357427215576172,
+                0.043469825744628904,
+                0.04353023910522461,
+                0.04353228759765625,
+                0.04357017517089844,
+                0.04354048156738281,
+                0.04357222366333008,
+                0.04342988967895508,
+                0.043561985015869144,
+                0.04352000045776367,
+                0.043512832641601565,
+                0.043340801239013675,
+                0.043410430908203124,
+                0.043630592346191405,
+                0.04350668716430664,
+                0.04342476654052734,
+                0.04359372711181641,
+                0.043633663177490234,
+                0.043614208221435545,
+                0.04370943832397461,
+                0.04362137603759766,
+                0.04351795196533203,
+                0.04354150390625,
+                0.0435865592956543,
+                0.04354662322998047,
+                0.043422721862792966,
+                0.04355379104614258,
+                0.04354355239868164,
+                0.04347903823852539,
+                0.043524097442626954,
+                0.04338790512084961,
+                0.04335718536376953,
+                0.04346060943603516,
+                0.04352102279663086,
+                0.04362547302246094,
+                0.04347903823852539,
+                0.043480064392089846,
+                0.043474945068359375,
+                0.04350668716430664,
+                0.04361011123657227,
+                0.04355788803100586,
+                0.04364287948608398,
+                0.04358041763305664,
+                0.04362444686889649,
+                0.04355686569213867,
+                0.04354048156738281,
+                0.0436121597290039,
+                0.04353433609008789,
+                0.04356710433959961,
+                0.04345446395874023,
+                0.04343500900268555,
+                0.0436121597290039,
+                0.043641857147216793,
+                0.043442176818847655,
+                0.04340019226074219,
+                0.043478015899658204,
+                0.04368281555175781,
+                0.043515903472900394,
+                0.04350259017944336,
+                0.04355686569213867,
+                0.04364287948608398,
+                0.043730945587158204,
+                0.043774974822998046,
+                0.04369203186035156,
+                0.04366950225830078,
+                0.043641857147216793,
+                0.04363673782348633,
+                0.043638782501220705,
+                0.04355276870727539,
+                0.04357222366333008,
+                0.043734016418457033,
+                0.04354764938354492,
+                0.04350371170043945,
+                0.04368783950805664,
+                0.04358041763305664,
+                0.04356095886230469,
+                0.04375961685180664,
+                0.04358758544921875,
+                0.04378009414672852,
+                0.04385996627807617,
+                0.04376166534423828,
+                0.04369715118408203,
+                0.04374835205078125,
+                0.043630592346191405,
+                0.04372582244873047,
+                0.043894783020019534,
+                0.04357120132446289,
+                0.043579391479492184,
+                0.04362444686889649,
+                0.043579391479492184,
+                0.04376268768310547,
+                0.043758590698242186,
+                0.04377804946899414,
+                0.04376166534423828,
+                0.04371148681640625,
+                0.04377395248413086,
+                0.043796478271484376,
+                0.043720703125,
+                0.04374835205078125,
+                0.04365824127197265,
+                0.04379033660888672,
+                0.04374118423461914,
+                0.04387942504882812,
+                0.043757568359375,
+                0.0435865592956543,
+                0.04372889709472656,
+                0.04371865463256836,
+                0.043633663177490234,
+                0.04374220657348633,
+                0.04372480010986328,
+                0.04391424179077148,
+                0.04369203186035156,
+                0.04375449752807617,
+                0.04379955291748047,
+                0.04373299026489258,
+                0.04365311813354492,
+                0.04363673782348633,
+                0.04376678466796875,
+                0.04361625671386719,
+                0.04378726577758789,
+                0.04370534515380859,
+                0.043817985534667966,
+                0.043630592346191405,
+                0.043597824096679685,
+                0.04364595031738281,
+                0.04365926361083984,
+                0.04360396957397461,
+                0.04358041763305664,
+                0.04362854385375976,
+                0.04376166534423828,
+                0.04358246231079101,
+                0.043663360595703124,
+                0.04359884643554687,
+                0.04379238510131836,
+                0.04371353530883789,
+                0.043671550750732424,
+                0.04377088165283203,
+                0.04381081771850586,
+                0.04386918258666992,
+                0.04382515335083008,
+                0.04376166534423828,
+                0.04360806274414063,
+                0.04368998336791992,
+                0.043602943420410156,
+                0.043663360595703124,
+                0.04379852676391602,
+                0.04358246231079101,
+                0.04381388854980469,
+                0.04373196792602539,
+                0.04357529449462891,
+                0.043650047302246094,
+                0.0436234245300293,
+                0.04342476654052734,
+                0.04354662322998047,
+                0.04345139312744141,
+                0.04366950225830078,
+                0.043589630126953126,
+                0.04370431900024414,
+                0.04357120132446289,
+                0.043514881134033206,
+                0.043633663177490234,
+                0.04366233444213867,
+                0.04363980865478516,
+                0.043655166625976564,
+                0.04353433609008789,
+                0.04361523056030273,
+                0.04374745559692383,
+                0.043559806823730465,
+                0.043622398376464845,
+                0.04359884643554687,
+                0.04350668716430664,
+                0.043448318481445314,
+                0.04370841598510742,
+                0.04367769622802734,
+                0.04369305419921875,
+                0.043622398376464845,
+                0.04373811340332031,
+                0.043687934875488284,
+                0.04364297485351563,
+                0.04366019058227539,
+                0.04382310485839844,
+                0.04363161468505859,
+                0.04380057525634766,
+                0.04377190399169922,
+                0.043911167144775394,
+                0.04367871856689453,
+                0.04356403350830078,
+                0.04352511978149414,
+                0.04358860778808594,
+                0.043640830993652346,
+                0.043663360595703124,
+                0.04368281555175781,
+                0.043753471374511715,
+                0.04355583953857422,
+                0.04349747085571289,
+                0.04351078414916992,
+                0.0437391357421875,
+                0.04367462539672851,
+                0.04359987258911133,
+                0.04371353530883789,
+                0.04353023910522461,
+                0.043670528411865236,
+                0.043602943420410156,
+                0.04341145706176758,
+                0.043638782501220705,
+                0.04379955291748047,
+                0.04360704040527344,
+                0.0436940803527832,
+                0.04360396957397461,
+                0.04355276870727539,
+                0.04373503875732422,
+                0.043535358428955076,
+                0.04358041763305664,
+                0.04369100952148437,
+                0.0436234245300293,
+                0.04373503875732422,
+                0.04369612884521484,
+                0.04373196792602539,
+                0.04375551986694336,
+                0.0435968017578125,
+                0.043717632293701174,
+                0.043720703125,
+                0.043676673889160154,
+                0.0436234245300293,
+                0.04373811340332031,
+                0.04362956619262695,
+                0.04354150390625,
+                0.04334284973144531,
+                0.043635711669921876,
+                0.04372684860229492,
+                0.04368281555175781,
+                0.04356915283203125,
+                0.04379852676391602,
+                0.043848705291748044,
+                0.04371558380126953,
+                0.043843582153320314,
+                0.04369919967651367,
+                0.04359270477294922,
+                0.04357734298706055,
+                0.0435230712890625,
+                0.043545600891113284,
+                0.043687934875488284
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 22.91947882641629
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}