Upload /phi-2/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

phi-2/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,795 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1032.773632,
+            "max_vram": 2600.46848,
+            "max_reserved": 2116.026368,
+            "max_allocated": 2014.912
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.04525479243351867,
+            "stdev": 0.0002673932631089354,
+            "values": [
+                0.04769750213623047,
+                0.046266368865966793,
+                0.04585062408447266,
+                0.04586611175537109,
+                0.04574720001220703,
+                0.045742080688476565,
+                0.045972545623779296,
+                0.04586393737792969,
+                0.04584447860717773,
+                0.04611276626586914,
+                0.04582297515869141,
+                0.04596736145019531,
+                0.04566835021972656,
+                0.04543590545654297,
+                0.04508364868164062,
+                0.04523116683959961,
+                0.04512176132202148,
+                0.04513689422607422,
+                0.04514406585693359,
+                0.04519424057006836,
+                0.04528025436401367,
+                0.04521984100341797,
+                0.04566352081298828,
+                0.04522086334228516,
+                0.04539187240600586,
+                0.04524544143676758,
+                0.04502732849121094,
+                0.04526182556152344,
+                0.04517068862915039,
+                0.04522188949584961,
+                0.04516352081298828,
+                0.0451033935546875,
+                0.045093887329101565,
+                0.04523519897460938,
+                0.04521267318725586,
+                0.045461505889892576,
+                0.04526387023925781,
+                0.045257728576660154,
+                0.04519948959350586,
+                0.04526899337768555,
+                0.04518092727661133,
+                0.045456382751464845,
+                0.04512768173217773,
+                0.045276161193847655,
+                0.04526899337768555,
+                0.04577177429199219,
+                0.04530688095092773,
+                0.04529782485961914,
+                0.045230079650878906,
+                0.04520652770996094,
+                0.045088768005371094,
+                0.04535532760620117,
+                0.045156352996826174,
+                0.04520662307739258,
+                0.04513075256347656,
+                0.04528332901000977,
+                0.04505606460571289,
+                0.04516556930541992,
+                0.045000705718994144,
+                0.045146110534667966,
+                0.04499148941040039,
+                0.04515350341796875,
+                0.04512870407104492,
+                0.045042686462402344,
+                0.04510528182983398,
+                0.045036544799804686,
+                0.04513177490234375,
+                0.04516044616699219,
+                0.04577382278442383,
+                0.04530483245849609,
+                0.04499353790283203,
+                0.045159423828125,
+                0.04515737533569336,
+                0.04512768173217773,
+                0.045197311401367186,
+                0.04531916809082031,
+                0.04516147232055664,
+                0.04521984100341797,
+                0.04506828689575195,
+                0.045200382232666016,
+                0.04515532684326172,
+                0.045142017364501956,
+                0.04504787063598633,
+                0.045127872467041016,
+                0.045020416259765626,
+                0.04516454315185547,
+                0.045162494659423826,
+                0.04514918518066406,
+                0.045475841522216794,
+                0.04514406585693359,
+                0.04531507110595703,
+                0.04535603332519531,
+                0.04503472137451172,
+                0.04508160018920898,
+                0.04510134506225586,
+                0.04525875091552734,
+                0.04510515213012695,
+                0.0451328010559082,
+                0.04505190277099609,
+                0.045080577850341794,
+                0.0451778564453125,
+                0.04511539077758789,
+                0.04512153625488281,
+                0.04508467102050781,
+                0.04503142547607422,
+                0.045080577850341794,
+                0.0450252799987793,
+                0.04503350448608399,
+                0.045568000793457034,
+                0.0452147216796875,
+                0.045052928924560545,
+                0.045129726409912106,
+                0.04512153625488281,
+                0.045085697174072265,
+                0.045123584747314455,
+                0.0451512336730957,
+                0.045125694274902345,
+                0.04520959854125976,
+                0.04507852935791016,
+                0.045208576202392575,
+                0.04521984100341797,
+                0.04532633590698242,
+                0.04506521606445312,
+                0.04523212814331055,
+                0.04524544143676758,
+                0.04516761779785156,
+                0.04550688171386719,
+                0.045284351348876956,
+                0.04541132736206055,
+                0.04544515228271485,
+                0.045445121765136716,
+                0.04537036895751953,
+                0.04528332901000977,
+                0.04542464065551758,
+                0.04545945739746094,
+                0.04536627197265625,
+                0.045246654510498044,
+                0.0452782096862793,
+                0.04518092727661133,
+                0.04521984100341797,
+                0.045074432373046876,
+                0.045197311401367186,
+                0.04506009674072266,
+                0.045134849548339843,
+                0.04519424057006836,
+                0.04563353729248047,
+                0.04530790328979492,
+                0.045402111053466795,
+                0.04521984100341797,
+                0.04516352081298828,
+                0.045172737121582034,
+                0.045058048248291016,
+                0.044984321594238284,
+                0.04516659164428711,
+                0.04516966247558594,
+                0.04526182556152344,
+                0.04533248138427735,
+                0.045399040222167966,
+                0.045174785614013675,
+                0.04521574401855469,
+                0.045178878784179685,
+                0.045211647033691404,
+                0.04510822296142578,
+                0.04518912124633789,
+                0.0451512336730957,
+                0.045644798278808595,
+                0.04522086334228516,
+                0.04531302261352539,
+                0.04532633590698242,
+                0.04547174453735352,
+                0.04517171096801758,
+                0.0451778564453125,
+                0.0452147216796875,
+                0.04529151916503906,
+                0.045195262908935545,
+                0.04524748611450195,
+                0.04507852935791016,
+                0.04513075256347656,
+                0.045126655578613284,
+                0.04534067153930664,
+                0.04500787353515625,
+                0.04520550537109375,
+                0.04511334228515625,
+                0.045251583099365236,
+                0.0451962890625,
+                0.04506828689575195,
+                0.04491980743408203,
+                0.04506316757202149,
+                0.045464576721191405,
+                0.04513894271850586,
+                0.045338623046875,
+                0.04506531143188477,
+                0.045093887329101565,
+                0.04513587188720703,
+                0.04507449722290039,
+                0.04512998580932617,
+                0.04513177490234375,
+                0.04516044616699219,
+                0.045146110534667966,
+                0.04518707275390625,
+                0.04540825653076172,
+                0.04550451278686524,
+                0.04509183883666992,
+                0.04512051010131836,
+                0.04513177490234375,
+                0.04518534469604492,
+                0.04521062469482422,
+                0.045159423828125,
+                0.04519935989379883,
+                0.04515145492553711,
+                0.04509183883666992,
+                0.045208576202392575,
+                0.04518707275390625,
+                0.045228160858154294,
+                0.045486080169677735,
+                0.04542464065551758,
+                0.04516352081298828,
+                0.04513689422607422,
+                0.04506419372558594,
+                0.04517375946044922,
+                0.04506726455688476
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 5656.859444799698
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1052.696576,
+            "max_vram": 2883.584,
+            "max_reserved": 2399.141888,
+            "max_allocated": 2328.542208
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 9.05663280868531,
+            "stdev": 0.007327739715575987,
+            "values": [
+                9.049305068969733,
+                9.063960548400885
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 28.156159732506218
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.035516207092883556,
+            "stdev": 0.00013792575206233963,
+            "values": [
+                0.03604582214355469,
+                0.03559936141967773,
+                0.03527999877929688,
+                0.03550809478759766,
+                0.035329025268554685,
+                0.03522457504272461,
+                0.03538022232055664,
+                0.03539763259887695,
+                0.03536281585693359,
+                0.03522252655029297,
+                0.03583180618286133,
+                0.03533004760742187,
+                0.035389438629150394,
+                0.0353177604675293,
+                0.035332096099853515,
+                0.03527577590942383,
+                0.035225601196289064,
+                0.03528406524658203,
+                0.03524192047119141,
+                0.03521532821655273,
+                0.0354703369140625,
+                0.035248126983642575,
+                0.035280895233154294,
+                0.03524607849121094,
+                0.03532086563110352,
+                0.03535663986206055,
+                0.035192832946777344,
+                0.035326976776123044,
+                0.03531980895996094,
+                0.03546623992919922,
+                0.035492862701416016,
+                0.03554816055297851,
+                0.03530342483520508,
+                0.03537715148925781,
+                0.035318782806396484,
+                0.03534950256347656,
+                0.03541708755493164,
+                0.03543142318725586,
+                0.035353599548339845,
+                0.035315711975097655,
+                0.03529420852661133,
+                0.035604480743408204,
+                0.03534758377075195,
+                0.03545587158203125,
+                0.0353259506225586,
+                0.03540787124633789,
+                0.03544166564941406,
+                0.035389438629150394,
+                0.035369983673095705,
+                0.03536588668823242,
+                0.03529318237304688,
+                0.0357212142944336,
+                0.03547955322265625,
+                0.03559423828125,
+                0.035460094451904296,
+                0.03544985580444336,
+                0.03536383819580078,
+                0.03577036666870117,
+                0.03569868850708008,
+                0.03524198532104492,
+                0.03531366348266601,
+                0.03537100982666016,
+                0.035672065734863284,
+                0.03546112060546875,
+                0.03552153778076172,
+                0.0355153923034668,
+                0.03540787124633789,
+                0.035350528717041016,
+                0.035345409393310545,
+                0.035525630950927735,
+                0.03539763259887695,
+                0.03550310516357422,
+                0.035762176513671876,
+                0.035535873413085936,
+                0.03553279876708984,
+                0.035432449340820314,
+                0.03561779022216797,
+                0.03581951904296875,
+                0.03561369705200195,
+                0.03558707046508789,
+                0.03585331344604492,
+                0.03556681442260742,
+                0.03562473678588867,
+                0.035566593170166014,
+                0.03551846313476562,
+                0.03548672103881836,
+                0.03573555374145508,
+                0.03535257720947266,
+                0.03546214294433594,
+                0.03539353561401367,
+                0.03564652633666992,
+                0.03553887939453125,
+                0.03554508972167969,
+                0.035484672546386715,
+                0.03564857482910156,
+                0.03560236740112305,
+                0.03551232147216797,
+                0.03551846313476562,
+                0.03578879928588867,
+                0.03551641464233399,
+                0.03555327987670898,
+                0.03540991973876953,
+                0.03553792190551758,
+                0.03540582275390625,
+                0.035372032165527346,
+                0.03543040084838867,
+                0.035631103515625,
+                0.03550822448730469,
+                0.03550207901000976,
+                0.035460094451904296,
+                0.03538534545898438,
+                0.03544575881958008,
+                0.03558297729492187,
+                0.03557580947875977,
+                0.03560675048828125,
+                0.0354639663696289,
+                0.03578777694702148,
+                0.03564646530151367,
+                0.03543040084838867,
+                0.03547238540649414,
+                0.035538944244384765,
+                0.035558399200439454,
+                0.03553279876708984,
+                0.03548262405395508,
+                0.03545087814331055,
+                0.03544268798828125,
+                0.03543756866455078,
+                0.035422206878662106,
+                0.03577241516113281,
+                0.03583283233642578,
+                0.03570995330810547,
+                0.03556966400146484,
+                0.035432449340820314,
+                0.035593215942382815,
+                0.03559526443481445,
+                0.035517440795898435,
+                0.03560857772827149,
+                0.03546316909790039,
+                0.03554304122924805,
+                0.03543552017211914,
+                0.03552870559692383,
+                0.035813377380371096,
+                0.03584928131103516,
+                0.035643329620361326,
+                0.03555635070800781,
+                0.03555737686157227,
+                0.03544473648071289,
+                0.035552257537841796,
+                0.03570995330810547,
+                0.03555942535400391,
+                0.035566593170166014,
+                0.035383296966552735,
+                0.03560550308227539,
+                0.035509246826171875,
+                0.0355153923034668,
+                0.03557603073120117,
+                0.035612449645996094,
+                0.03548672103881836,
+                0.035454975128173825,
+                0.035746814727783204,
+                0.0354969596862793,
+                0.03545907211303711,
+                0.03547955322265625,
+                0.03558707046508789,
+                0.035454975128173825,
+                0.03544063949584961,
+                0.03548672103881836,
+                0.035386367797851564,
+                0.03553484725952148,
+                0.0353331184387207,
+                0.03553084945678711,
+                0.03547024154663086,
+                0.035547134399414065,
+                0.03538739013671875,
+                0.03540070343017578,
+                0.035302398681640625,
+                0.03546931076049804,
+                0.03527577590942383,
+                0.03549491119384766,
+                0.035279872894287106,
+                0.035386367797851564,
+                0.0353331184387207,
+                0.03530035018920898,
+                0.03563724899291992,
+                0.03529638290405274,
+                0.03533606338500977,
+                0.03545702362060547,
+                0.035345409393310545,
+                0.03540079879760742,
+                0.035297183990478515,
+                0.03522662353515625,
+                0.03529216003417969,
+                0.03569049453735352,
+                0.03554304122924805,
+                0.03542732620239258,
+                0.035519489288330076,
+                0.03565670394897461,
+                0.03546931076049804,
+                0.0356577262878418,
+                0.035759105682373046,
+                0.035694591522216795,
+                0.0354785270690918,
+                0.03545087814331055,
+                0.03546931076049804,
+                0.03545907211303711,
+                0.035514366149902346,
+                0.035517440795898435,
+                0.03559628677368164,
+                0.03538431930541992,
+                0.035784702301025394,
+                0.03564134216308594,
+                0.03550822448730469,
+                0.03559423828125,
+                0.035591167449951173,
+                0.03544579315185547,
+                0.03555324935913086,
+                0.0354785270690918,
+                0.03557068634033203,
+                0.0354252815246582,
+                0.035460094451904296,
+                0.03539046478271484,
+                0.03547340774536133,
+                0.035519489288330076,
+                0.035465217590332034,
+                0.03571814346313477,
+                0.035501056671142575,
+                0.0355491828918457,
+                0.03560243225097656,
+                0.035356670379638674,
+                0.03548160171508789,
+                0.035379360198974606,
+                0.035401569366455075,
+                0.035525630950927735,
+                0.03549491119384766,
+                0.03543555068969727,
+                0.03553788757324219,
+                0.035538944244384765,
+                0.03537612915039062,
+                0.035418113708496096,
+                0.03545600128173828,
+                0.03543961715698242,
+                0.035776512145996094,
+                0.035550209045410154,
+                0.03550003051757813,
+                0.03552870559692383,
+                0.035573760986328126,
+                0.035514366149902346,
+                0.03561062240600586,
+                0.0354252815246582,
+                0.03544575881958008,
+                0.03551027297973633,
+                0.0354969596862793,
+                0.035373054504394534,
+                0.03541299057006836,
+                0.03558195114135742,
+                0.035151870727539065,
+                0.035448833465576174,
+                0.03551641464233399,
+                0.03542015838623047,
+                0.0354785270690918,
+                0.03546419143676758,
+                0.035418113708496096,
+                0.03534745788574219,
+                0.035454975128173825,
+                0.03520204925537109,
+                0.03567103958129883,
+                0.03549900817871094,
+                0.035767295837402346,
+                0.035576000213623046,
+                0.035480384826660154,
+                0.03538534545898438,
+                0.035558399200439454,
+                0.03548876953125,
+                0.03549900817871094,
+                0.035501056671142575,
+                0.03568742370605469,
+                0.03554611206054688,
+                0.035631103515625,
+                0.035514366149902346,
+                0.03552358245849609,
+                0.03553484725952148,
+                0.035683326721191407,
+                0.035675136566162106,
+                0.03540172958374024,
+                0.03532185745239258,
+                0.03577139282226562,
+                0.03546828842163086,
+                0.0354856948852539,
+                0.03549798583984375,
+                0.035504127502441404,
+                0.035544063568115236,
+                0.03562700653076172,
+                0.03548057556152344,
+                0.03561574554443359,
+                0.03546726226806641,
+                0.035904510498046875,
+                0.03552358245849609,
+                0.03555430221557617,
+                0.03567103958129883,
+                0.03566080093383789,
+                0.0355860481262207,
+                0.035517440795898435,
+                0.03551027297973633,
+                0.03537408065795898,
+                0.03558911895751953,
+                0.0355555191040039,
+                0.03580601501464844,
+                0.03546726226806641,
+                0.03563113784790039,
+                0.03567305755615234,
+                0.03583692932128906,
+                0.03552460861206055,
+                0.03561881637573242,
+                0.03550003051757813,
+                0.03555430221557617,
+                0.035563518524169925,
+                0.035737598419189456,
+                0.035604480743408204,
+                0.03559423828125,
+                0.03562700653076172,
+                0.03544678497314453,
+                0.03539251327514648,
+                0.03553279876708984,
+                0.0354856948852539,
+                0.03552665710449219,
+                0.035789825439453124,
+                0.035525630950927735,
+                0.03562192153930664,
+                0.03547849655151367,
+                0.035560447692871096,
+                0.035609600067138675,
+                0.03549593734741211,
+                0.03559423828125,
+                0.03570687866210937,
+                0.03549798583984375,
+                0.03555327987670898,
+                0.035547134399414065,
+                0.035552257537841796,
+                0.03585843276977539,
+                0.03540582275390625,
+                0.0354150390625,
+                0.03552665710449219,
+                0.03555123138427734,
+                0.03548364639282227,
+                0.035622913360595705,
+                0.03543961715698242,
+                0.035507198333740234,
+                0.03552665710449219,
+                0.03555430221557617,
+                0.03547750473022461,
+                0.03551641464233399,
+                0.03548364639282227,
+                0.03539558410644531,
+                0.03548262405395508,
+                0.03572326278686523,
+                0.035606529235839846,
+                0.035525630950927735,
+                0.03554508972167969,
+                0.035533824920654294,
+                0.03547750473022461,
+                0.03544063949584961,
+                0.03548160171508789,
+                0.03549388885498047,
+                0.03560140609741211,
+                0.03553177642822265,
+                0.03545087814331055,
+                0.035745792388916016,
+                0.03555635070800781,
+                0.03546316909790039,
+                0.03547955322265625,
+                0.03545702362060547,
+                0.035625984191894534,
+                0.03584716796875,
+                0.03556454467773437,
+                0.03563827133178711,
+                0.03561471939086914,
+                0.035574783325195314,
+                0.03537510299682617,
+                0.03547545623779297,
+                0.03538227081298828,
+                0.035484672546386715,
+                0.03557273483276367,
+                0.035846145629882815,
+                0.03566284942626953,
+                0.035522560119628906,
+                0.03554508972167969,
+                0.03563520050048828,
+                0.03557580947875977,
+                0.03552665710449219,
+                0.035694591522216795,
+                0.035552257537841796,
+                0.03553177642822265,
+                0.03597619247436523,
+                0.03564031982421875,
+                0.03588198471069336,
+                0.03562700653076172,
+                0.035504127502441404,
+                0.03558297729492187,
+                0.035525630950927735,
+                0.035631103515625,
+                0.0356126708984375,
+                0.03560140609741211,
+                0.03558707046508789,
+                0.03582566452026367,
+                0.035705856323242184,
+                0.03557900619506836,
+                0.03559718322753906,
+                0.03559219360351563,
+                0.03566796875,
+                0.0355860481262207,
+                0.035602592468261716,
+                0.035630943298339844,
+                0.03573555374145508,
+                0.035917823791503906,
+                0.03568230438232422,
+                0.03566284942626953,
+                0.035576831817626955,
+                0.035631103515625,
+                0.03565670394897461,
+                0.03569049453735352,
+                0.035571712493896485,
+                0.03564646530151367,
+                0.03579391860961914,
+                0.03559423828125,
+                0.0355788803100586,
+                0.03551641464233399,
+                0.035560447692871096,
+                0.03580723190307617,
+                0.03570380783081055,
+                0.035568641662597655,
+                0.03559219360351563,
+                0.03578777694702148,
+                0.03558707046508789,
+                0.03552153778076172,
+                0.035489791870117186,
+                0.035620864868164064,
+                0.03543552017211914,
+                0.0354856948852539,
+                0.03544473648071289,
+                0.03543552017211914,
+                0.03544473648071289,
+                0.03550003051757813,
+                0.03542425537109375,
+                0.03536793518066406,
+                0.03538739013671875,
+                0.03540172958374024,
+                0.035699710845947266,
+                0.035507198333740234,
+                0.035402751922607424,
+                0.03561062240600586,
+                0.035566593170166014,
+                0.03559526443481445,
+                0.035356670379638674,
+                0.03544575881958008,
+                0.03544371032714844,
+                0.035535873413085936,
+                0.035530750274658206,
+                0.03540889739990234,
+                0.03560140609741211,
+                0.035484672546386715,
+                0.03544268798828125,
+                0.035386367797851564,
+                0.03540787124633789,
+                0.0352911376953125,
+                0.035340286254882815,
+                0.03543040084838867,
+                0.03543654251098633,
+                0.03536383819580078,
+                0.03552665710449219,
+                0.03539353561401367,
+                0.03585740661621094,
+                0.03552870559692383,
+                0.03557273483276367,
+                0.03555123138427734,
+                0.03562188720703125,
+                0.03544473648071289,
+                0.035509246826171875,
+                0.03541196823120117,
+                0.035694591522216795,
+                0.03551232147216797,
+                0.03531468963623047,
+                0.03533004760742187,
+                0.035432449340820314,
+                0.0353259506225586,
+                0.03545292663574219,
+                0.035438591003417966,
+                0.035386367797851564,
+                0.03578879928588867,
+                0.03540070343017578,
+                0.035351551055908204,
+                0.03546726226806641,
+                0.03543756866455078,
+                0.03558399963378906,
+                0.03550515365600586,
+                0.03549900817871094,
+                0.035476478576660156,
+                0.03550207901000976,
+                0.03539558410644531,
+                0.03558092880249023,
+                0.035389438629150394,
+                0.03545600128173828,
+                0.03532287979125977,
+                0.03593625640869141,
+                0.035448833465576174,
+                0.03547340774536133,
+                0.03547340774536133,
+                0.0356126708984375,
+                0.03567103958129883,
+                0.0353331184387207,
+                0.035454975128173825
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 28.156159732506225
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}