diff --git "a/ndarray-cache-b16.json" "b/ndarray-cache-b16.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache-b16.json" @@ -0,0 +1,4775 @@ +{ + "metadata": { + "ParamSize": 357, + "ParamBytes": 4828053504.0, + "BitsPerParam": 5.002305991918684 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 311164928, + "records": [ + { + "name": "lm_head.q_weight", + "shape": [ + 151936, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 311164928, + "byteOffset": 0 + } + ], + "md5sum": "aa88d7caf0fb972716d41f7ef8bcc436" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 38895616, + "records": [ + { + "name": "lm_head.q_scale", + "shape": [ + 151936, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 38895616, + "byteOffset": 0 + } + ], + "md5sum": "ee95b8d589492e38bbb50958af0e7838" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.0.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "cf42829973232dfac3b922a08547b656" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 28540928, + "records": [ + { + "name": "transformer.h.31.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + }, + { + "name": "transformer.h.31.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 22544384 + }, + { + "name": "transformer.ln_f.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 25362432 + }, + { + "name": "transformer.h.0.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 25370624 + }, + { + "name": "transformer.h.0.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 25395200 + } + ], + "md5sum": "11af9f3ecfdac67a8089550fea1a4cd8" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 31997952, + "records": [ + { + "name": "transformer.h.0.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.0.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.0.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.0.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.0.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 9453568 + } + ], + "md5sum": "300b1802c19bc9bad6a4331fcddd9d7e" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.0.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "51f9275d1a456ab6672b77b50654eb8d" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.1.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "9fbd60ebad38a9f1f65a585d220785a4" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.1.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "ff7080c14a7ee2f2b610657d5398e938" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 311164928, + "records": [ + { + "name": "transformer.wte.q_weight", + "shape": [ + 151936, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 311164928, + "byteOffset": 0 + } + ], + "md5sum": "c4a99ff17277f2a55c8b46fd37493a4e" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 38895616, + "records": [ + { + "name": "transformer.wte.q_scale", + "shape": [ + 151936, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 38895616, + "byteOffset": 0 + } + ], + "md5sum": "13eabdf16fabb76250ef07807c9e4cab" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.1.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "751f47c76e94c4cbfabc6e433013f7cd" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.2.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "a18ea06e44c8a84b86e3cb7c89d4e82e" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 32702464, + "records": [ + { + "name": "transformer.h.0.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 0 + }, + { + "name": "transformer.h.0.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 2818048 + }, + { + "name": "transformer.h.1.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 8454144 + }, + { + "name": "transformer.h.1.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 8478720 + }, + { + "name": "transformer.h.1.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 11624448 + }, + { + "name": "transformer.h.1.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 20013056 + }, + { + "name": "transformer.h.1.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 21061632 + }, + { + "name": "transformer.h.1.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 21069824 + }, + { + "name": "transformer.h.1.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.1.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 26714112 + }, + { + "name": "transformer.h.2.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 29532160 + }, + { + "name": "transformer.h.2.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 29556736 + } + ], + "md5sum": "b67efdcc08b8a3d97a68a56420fd4350" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 31997952, + "records": [ + { + "name": "transformer.h.2.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.2.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.2.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.2.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.2.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 9453568 + } + ], + "md5sum": "179d46e2a595938f2224164e8f6dc7dd" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.2.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "201a56aaa49e645395a49d857e9a82c4" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.3.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "629db4e22cf6348c04108c0bc82daa5f" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.3.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "71bf76a4e0c2667979ad21a6eaba13a8" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.3.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "0cd196f12800fa6389379ffe57359f3d" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.4.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "f21312110c2a619fa270eb338c1e2a49" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 32702464, + "records": [ + { + "name": "transformer.h.2.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 0 + }, + { + "name": "transformer.h.2.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 2818048 + }, + { + "name": "transformer.h.3.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 8454144 + }, + { + "name": "transformer.h.3.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 8478720 + }, + { + "name": "transformer.h.3.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 11624448 + }, + { + "name": "transformer.h.3.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 20013056 + }, + { + "name": "transformer.h.3.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 21061632 + }, + { + "name": "transformer.h.3.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 21069824 + }, + { + "name": "transformer.h.3.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.3.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 23896064 + }, + { + "name": "transformer.h.4.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 29532160 + }, + { + "name": "transformer.h.4.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 29556736 + } + ], + "md5sum": "447704abb3116a2adc11115668e3b218" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 31997952, + "records": [ + { + "name": "transformer.h.4.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.4.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.4.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.4.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.4.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 9453568 + } + ], + "md5sum": "d6c739f5f53e7cae12c61283f18e373e" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.4.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "0f84dab52095e448f03c63a54aae06b6" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.5.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "9a06b4304527fb1c3421b1edcbaea61a" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.5.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "6e2e74c38e2c487ac65a71ab80add8ac" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.5.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "5187f79d3211544d9400733f2a41418f" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.6.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "f3c553b6e0176b5ddbbbce186783d372" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 32702464, + "records": [ + { + "name": "transformer.h.4.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 0 + }, + { + "name": "transformer.h.4.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 2818048 + }, + { + "name": "transformer.h.5.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 8454144 + }, + { + "name": "transformer.h.5.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 8478720 + }, + { + "name": "transformer.h.5.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 11624448 + }, + { + "name": "transformer.h.5.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 20013056 + }, + { + "name": "transformer.h.5.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 21061632 + }, + { + "name": "transformer.h.5.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 21069824 + }, + { + "name": "transformer.h.5.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.5.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 23896064 + }, + { + "name": "transformer.h.6.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 29532160 + }, + { + "name": "transformer.h.6.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 29556736 + } + ], + "md5sum": "5de58143deae685f72a34258cf227eb4" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.6.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "a9664450379a0e0a289334d6fc05c2ff" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.10.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "de828059f2ca164daf7b0237b7997a30" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.10.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "832ead58ab1ba002ff08b9fa8b83e4eb" + }, + { + "dataPath": "params_shard_30.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.10.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "c11a16dff829c70794a77efd9119cb4e" + }, + { + "dataPath": "params_shard_31.bin", + "format": "raw-shard", + "nbytes": 30531584, + "records": [ + { + "name": "transformer.h.6.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.6.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.6.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.6.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.6.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 9453568 + }, + { + "name": "transformer.h.10.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 15089664 + }, + { + "name": "transformer.h.10.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 15114240 + }, + { + "name": "transformer.h.10.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18259968 + }, + { + "name": "transformer.h.10.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 26648576 + }, + { + "name": "transformer.h.10.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 27697152 + }, + { + "name": "transformer.h.10.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 27705344 + }, + { + "name": "transformer.h.10.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 27713536 + } + ], + "md5sum": "e3eeb9a3a7d838e24d1bc939c43b8ef9" + }, + { + "dataPath": "params_shard_32.bin", + "format": "raw-shard", + "nbytes": 30826496, + "records": [ + { + "name": "transformer.h.10.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 0 + }, + { + "name": "transformer.h.11.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 5636096 + }, + { + "name": "transformer.h.11.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 5660672 + } + ], + "md5sum": "bec565a26e88a3d81bbb0305a4968eed" + }, + { + "dataPath": "params_shard_33.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.11.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "5d1906ab0d3bca27f11ecd60992f6e30" + }, + { + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.6.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "3f2c2c84496ef50c28c051edeb11209e" + }, + { + "dataPath": "params_shard_35.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.7.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "003078224e454658a3c30198cc270601" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 32612352, + "records": [ + { + "name": "transformer.h.11.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 0 + }, + { + "name": "transformer.h.11.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 3145728 + }, + { + "name": "transformer.h.11.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 11534336 + }, + { + "name": "transformer.h.11.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.11.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12591104 + }, + { + "name": "transformer.h.11.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 12599296 + }, + { + "name": "transformer.h.6.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 18235392 + }, + { + "name": "transformer.h.7.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 21053440 + }, + { + "name": "transformer.h.7.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.7.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 24223744 + } + ], + "md5sum": "7a5c9122b371b899c0a7405590a3bfba" + }, + { + "dataPath": "params_shard_37.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.7.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "c350065115027599298a2cfb814589e4" + }, + { + "dataPath": "params_shard_38.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.8.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "87e39e97449adfa62cff75978dc183c9" + }, + { + "dataPath": "params_shard_39.bin", + "format": "raw-shard", + "nbytes": 32088064, + "records": [ + { + "name": "transformer.h.7.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 0 + }, + { + "name": "transformer.h.7.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1048576 + }, + { + "name": "transformer.h.7.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1056768 + }, + { + "name": "transformer.h.7.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 1064960 + }, + { + "name": "transformer.h.7.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 23609344 + }, + { + "name": "transformer.h.7.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 26427392 + }, + { + "name": "transformer.h.8.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 32063488 + } + ], + "md5sum": "a423406111b02fe9d3fd6b1e0125e27e" + }, + { + "dataPath": "params_shard_40.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.8.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "ec0bf66d65ec5d62d046c6316e344219" + }, + { + "dataPath": "params_shard_41.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.8.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "5f554cc6cf090257218b2ccd2cd7a2c7" + }, + { + "dataPath": "params_shard_42.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.9.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "005f16699e0695f3477108b4878775db" + }, + { + "dataPath": "params_shard_43.bin", + "format": "raw-shard", + "nbytes": 32612352, + "records": [ + { + "name": "transformer.h.8.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 0 + }, + { + "name": "transformer.h.8.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 3145728 + }, + { + "name": "transformer.h.8.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 11534336 + }, + { + "name": "transformer.h.8.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.8.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12591104 + }, + { + "name": "transformer.h.8.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 12599296 + }, + { + "name": "transformer.h.8.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 15417344 + }, + { + "name": "transformer.h.9.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 21053440 + }, + { + "name": "transformer.h.9.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.9.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 24223744 + } + ], + "md5sum": "6574228e8078052da7406d8e5c6e3323" + }, + { + "dataPath": "params_shard_44.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.9.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "ac26d5950f30b41873aaf8fdd31d8a5a" + }, + { + "dataPath": "params_shard_45.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.11.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "1b70eabfac49ca46cce21000ddd0e681" + }, + { + "dataPath": "params_shard_46.bin", + "format": "raw-shard", + "nbytes": 32063488, + "records": [ + { + "name": "transformer.h.9.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 0 + }, + { + "name": "transformer.h.9.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1048576 + }, + { + "name": "transformer.h.9.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1056768 + }, + { + "name": "transformer.h.9.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 1064960 + }, + { + "name": "transformer.h.9.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 23609344 + }, + { + "name": "transformer.h.9.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 26427392 + } + ], + "md5sum": "3c239a63bc814758c1ead6c3db7e2864" + }, + { + "dataPath": "params_shard_47.bin", + "format": "raw-shard", + "nbytes": 31154176, + "records": [ + { + "name": "transformer.h.11.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 0 + }, + { + "name": "transformer.h.12.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 2818048 + }, + { + "name": "transformer.h.12.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 2842624 + }, + { + "name": "transformer.h.12.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 28008448 + } + ], + "md5sum": "2d49c457c9fc0616e012b58123bc61da" + }, + { + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 31997952, + "records": [ + { + "name": "transformer.h.12.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.12.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.12.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.12.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.12.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 9453568 + } + ], + "md5sum": "63cf17b25bed7473a9d49f76e454836a" + }, + { + "dataPath": "params_shard_49.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.12.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "9ed41e6446bc625e4acb6554b0145e67" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.13.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "1f68e723aa3d9522ad946369510b0eb3" + }, + { + "dataPath": "params_shard_51.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.13.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "44b713cc18bce594c20ee9c723088c65" + }, + { + "dataPath": "params_shard_52.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.13.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "2e5ad2394dadd2e365e07495c8943cc3" + }, + { + "dataPath": "params_shard_53.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.14.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "045cc4034ab513764fb89b9c9056135b" + }, + { + "dataPath": "params_shard_54.bin", + "format": "raw-shard", + "nbytes": 32702464, + "records": [ + { + "name": "transformer.h.12.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 0 + }, + { + "name": "transformer.h.12.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 2818048 + }, + { + "name": "transformer.h.13.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 8454144 + }, + { + "name": "transformer.h.13.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 8478720 + }, + { + "name": "transformer.h.13.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 11624448 + }, + { + "name": "transformer.h.13.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 20013056 + }, + { + "name": "transformer.h.13.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 21061632 + }, + { + "name": "transformer.h.13.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 21069824 + }, + { + "name": "transformer.h.13.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.13.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 23896064 + }, + { + "name": "transformer.h.14.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 29532160 + }, + { + "name": "transformer.h.14.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 29556736 + } + ], + "md5sum": "9fa9878d3d993ced462ef3a8e0126910" + }, + { + "dataPath": "params_shard_55.bin", + "format": "raw-shard", + "nbytes": 31997952, + "records": [ + { + "name": "transformer.h.14.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.14.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.14.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.14.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.14.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 9453568 + } + ], + "md5sum": "0a9516235055ba5f8b9e97ddcc9cfcce" + }, + { + "dataPath": "params_shard_56.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.14.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "848982c2349d61f222ebcd7153ce79fe" + }, + { + "dataPath": "params_shard_57.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.15.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "7fb6bddd37ffc818fc276d87b1887abd" + }, + { + "dataPath": "params_shard_58.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.15.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "786ca9e6d91621cc3de7796ea2cc8bba" + }, + { + "dataPath": "params_shard_59.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.15.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "874f9e33a8f500e79c6ff5dba102c5f2" + }, + { + "dataPath": "params_shard_60.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.16.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2722ec54508a04ee477f282e19d8a9c8" + }, + { + "dataPath": "params_shard_61.bin", + "format": "raw-shard", + "nbytes": 32702464, + "records": [ + { + "name": "transformer.h.14.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 0 + }, + { + "name": "transformer.h.14.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 2818048 + }, + { + "name": "transformer.h.15.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 8454144 + }, + { + "name": "transformer.h.15.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 8478720 + }, + { + "name": "transformer.h.15.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 11624448 + }, + { + "name": "transformer.h.15.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 20013056 + }, + { + "name": "transformer.h.15.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 21061632 + }, + { + "name": "transformer.h.15.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 21069824 + }, + { + "name": "transformer.h.15.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.15.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 23896064 + }, + { + "name": "transformer.h.16.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 29532160 + }, + { + "name": "transformer.h.16.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 29556736 + } + ], + "md5sum": "6526c3507085093a059b48fdad08d477" + }, + { + "dataPath": "params_shard_62.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.16.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "a89863cf3d70fddd3bb9d77c147f7509" + }, + { + "dataPath": "params_shard_63.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.16.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "9bff1a0a104bee15d4d6d84eb41203ae" + }, + { + "dataPath": "params_shard_64.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.17.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "225a9d5e502a2c4dcff26ab831f6eadf" + }, + { + "dataPath": "params_shard_65.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.17.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "1af2db3241a42ad525a2b97aa297848f" + }, + { + "dataPath": "params_shard_66.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.17.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "90765666c92e384df584955febd0b3e1" + }, + { + "dataPath": "params_shard_67.bin", + "format": "raw-shard", + "nbytes": 33349632, + "records": [ + { + "name": "transformer.h.16.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.16.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.16.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.16.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.16.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 9453568 + }, + { + "name": "transformer.h.16.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 15089664 + }, + { + "name": "transformer.h.17.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 17907712 + }, + { + "name": "transformer.h.17.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 17932288 + }, + { + "name": "transformer.h.17.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.17.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 29466624 + }, + { + "name": "transformer.h.17.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 30515200 + }, + { + "name": "transformer.h.17.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 30523392 + }, + { + "name": "transformer.h.17.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 30531584 + } + ], + "md5sum": "f4332d4ba633b43b227e1ac156d339a5" + }, + { + "dataPath": "params_shard_68.bin", + "format": "raw-shard", + "nbytes": 30826496, + "records": [ + { + "name": "transformer.h.17.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 0 + }, + { + "name": "transformer.h.18.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 5636096 + }, + { + "name": "transformer.h.18.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 5660672 + } + ], + "md5sum": "c0b74ec23aa408835e546ad6ca72f53b" + }, + { + "dataPath": "params_shard_69.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.18.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "461bbb84f65cd510ebb4eb2fabd26f27" + }, + { + "dataPath": "params_shard_70.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.18.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "61f6ad83f3165343208e5f74d9826c21" + }, + { + "dataPath": "params_shard_71.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.19.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "86f754b00141d8548a5be2211d86b323" + }, + { + "dataPath": "params_shard_72.bin", + "format": "raw-shard", + "nbytes": 32612352, + "records": [ + { + "name": "transformer.h.18.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 0 + }, + { + "name": "transformer.h.18.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 3145728 + }, + { + "name": "transformer.h.18.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 11534336 + }, + { + "name": "transformer.h.18.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.18.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12591104 + }, + { + "name": "transformer.h.18.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 12599296 + }, + { + "name": "transformer.h.18.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 15417344 + }, + { + "name": "transformer.h.19.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 21053440 + }, + { + "name": "transformer.h.19.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.19.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 24223744 + } + ], + "md5sum": "5df10301551fe5b592c64ffc462fd1a8" + }, + { + "dataPath": "params_shard_73.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.19.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "0fb469e9ecd68f1e323d5426dc966e12" + }, + { + "dataPath": "params_shard_74.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.20.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "df0bb7e0eeb50c0308d67ceadebb79d1" + }, + { + "dataPath": "params_shard_75.bin", + "format": "raw-shard", + "nbytes": 32088064, + "records": [ + { + "name": "transformer.h.19.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 0 + }, + { + "name": "transformer.h.19.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1048576 + }, + { + "name": "transformer.h.19.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1056768 + }, + { + "name": "transformer.h.19.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 1064960 + }, + { + "name": "transformer.h.19.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 23609344 + }, + { + "name": "transformer.h.19.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 26427392 + }, + { + "name": "transformer.h.20.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 32063488 + } + ], + "md5sum": "a5f946023ede44593f52234cb722c97e" + }, + { + "dataPath": "params_shard_76.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.20.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "bd9817249774537cb56020bf638b5033" + }, + { + "dataPath": "params_shard_77.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.20.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "cb099aad3594180b8026a4336ed75b55" + }, + { + "dataPath": "params_shard_78.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.21.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "c6f895d9c2fd122cb9067513d0c55648" + }, + { + "dataPath": "params_shard_79.bin", + "format": "raw-shard", + "nbytes": 32612352, + "records": [ + { + "name": "transformer.h.20.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 0 + }, + { + "name": "transformer.h.20.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 3145728 + }, + { + "name": "transformer.h.20.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 11534336 + }, + { + "name": "transformer.h.20.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.20.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12591104 + }, + { + "name": "transformer.h.20.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 12599296 + }, + { + "name": "transformer.h.20.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 15417344 + }, + { + "name": "transformer.h.21.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 21053440 + }, + { + "name": "transformer.h.21.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.21.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 24223744 + } + ], + "md5sum": "b61003364100442716a198e786869513" + }, + { + "dataPath": "params_shard_80.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.21.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "7e64bc1130809a4f94f42bf722d82849" + }, + { + "dataPath": "params_shard_81.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.22.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "3f9449019c6c24c0570c11fd8b109ec6" + }, + { + "dataPath": "params_shard_82.bin", + "format": "raw-shard", + "nbytes": 32088064, + "records": [ + { + "name": "transformer.h.21.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 0 + }, + { + "name": "transformer.h.21.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1048576 + }, + { + "name": "transformer.h.21.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1056768 + }, + { + "name": "transformer.h.21.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 1064960 + }, + { + "name": "transformer.h.21.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 6701056 + }, + { + "name": "transformer.h.21.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 29245440 + }, + { + "name": "transformer.h.22.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 32063488 + } + ], + "md5sum": "e43e58745f858ae056166aa1f4ce752c" + }, + { + "dataPath": "params_shard_83.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.22.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "f8a71003371cfe19031e292b4106af5d" + }, + { + "dataPath": "params_shard_84.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.22.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "39061f04107b1d1046252eb52ec1a802" + }, + { + "dataPath": "params_shard_85.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.23.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "d0ceed31a3c4425e1787042be765b34e" + }, + { + "dataPath": "params_shard_86.bin", + "format": "raw-shard", + "nbytes": 32612352, + "records": [ + { + "name": "transformer.h.22.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 0 + }, + { + "name": "transformer.h.22.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 3145728 + }, + { + "name": "transformer.h.22.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 11534336 + }, + { + "name": "transformer.h.22.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.22.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12591104 + }, + { + "name": "transformer.h.22.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 12599296 + }, + { + "name": "transformer.h.22.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 15417344 + }, + { + "name": "transformer.h.23.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 21053440 + }, + { + "name": "transformer.h.23.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.23.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 24223744 + } + ], + "md5sum": "31bb67658bbfd91061c094c824cdd5b1" + }, + { + "dataPath": "params_shard_87.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.23.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "d66cf3e4ee88bbff71886556773b7db8" + }, + { + "dataPath": "params_shard_88.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.24.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "52c1eba96743cf456c2b978483ed4975" + }, + { + "dataPath": "params_shard_89.bin", + "format": "raw-shard", + "nbytes": 32088064, + "records": [ + { + "name": "transformer.h.23.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 0 + }, + { + "name": "transformer.h.23.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1048576 + }, + { + "name": "transformer.h.23.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1056768 + }, + { + "name": "transformer.h.23.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 1064960 + }, + { + "name": "transformer.h.23.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 23609344 + }, + { + "name": "transformer.h.23.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 26427392 + }, + { + "name": "transformer.h.24.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 32063488 + } + ], + "md5sum": "1aa7657ca7c268ffee19e3c5493741d1" + }, + { + "dataPath": "params_shard_90.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.24.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "9817567b137aa023fdf3d373a0ce5c58" + }, + { + "dataPath": "params_shard_91.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.24.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "9d37d8b20e76057efd2ce625f6fecde1" + }, + { + "dataPath": "params_shard_92.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.25.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "26f552db11cb3f0f3f38a90ef9c98fe8" + }, + { + "dataPath": "params_shard_93.bin", + "format": "raw-shard", + "nbytes": 32612352, + "records": [ + { + "name": "transformer.h.24.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 0 + }, + { + "name": "transformer.h.24.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 3145728 + }, + { + "name": "transformer.h.24.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 11534336 + }, + { + "name": "transformer.h.24.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.24.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12591104 + }, + { + "name": "transformer.h.24.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 12599296 + }, + { + "name": "transformer.h.24.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 15417344 + }, + { + "name": "transformer.h.25.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 21053440 + }, + { + "name": "transformer.h.25.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.25.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 24223744 + } + ], + "md5sum": "3d3a09b55a78d7e0b74f6efced6a7150" + }, + { + "dataPath": "params_shard_94.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.25.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "1865351ec84e3e424214fbc812cdabd3" + }, + { + "dataPath": "params_shard_95.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.26.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "e80cc24ab0602c6a3841f648ee08268f" + }, + { + "dataPath": "params_shard_96.bin", + "format": "raw-shard", + "nbytes": 32088064, + "records": [ + { + "name": "transformer.h.25.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 0 + }, + { + "name": "transformer.h.25.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1048576 + }, + { + "name": "transformer.h.25.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1056768 + }, + { + "name": "transformer.h.25.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 1064960 + }, + { + "name": "transformer.h.25.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 23609344 + }, + { + "name": "transformer.h.25.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 26427392 + }, + { + "name": "transformer.h.26.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 32063488 + } + ], + "md5sum": "04094499a9067e473a58030eab14cd63" + }, + { + "dataPath": "params_shard_97.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.26.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "063a95d54ca1c0ac15a4e227ca867f98" + }, + { + "dataPath": "params_shard_98.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.26.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "8931643b778b2dc177430b2cd60a73a2" + }, + { + "dataPath": "params_shard_99.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.27.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "5bde556559c211bb854f33080af0a064" + }, + { + "dataPath": "params_shard_100.bin", + "format": "raw-shard", + "nbytes": 32612352, + "records": [ + { + "name": "transformer.h.26.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 0 + }, + { + "name": "transformer.h.26.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 3145728 + }, + { + "name": "transformer.h.26.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 11534336 + }, + { + "name": "transformer.h.26.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.26.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12591104 + }, + { + "name": "transformer.h.26.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 12599296 + }, + { + "name": "transformer.h.26.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 18235392 + }, + { + "name": "transformer.h.27.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 21053440 + }, + { + "name": "transformer.h.27.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.27.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 24223744 + } + ], + "md5sum": "5cdea8997515c273b43c67894387a1cd" + }, + { + "dataPath": "params_shard_101.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.27.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "ee7d6eb797a5a25fb4de0e204d502839" + }, + { + "dataPath": "params_shard_102.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.28.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "a7ad901d6a084bfd4f4d9057c2111fed" + }, + { + "dataPath": "params_shard_103.bin", + "format": "raw-shard", + "nbytes": 32088064, + "records": [ + { + "name": "transformer.h.27.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 0 + }, + { + "name": "transformer.h.27.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1048576 + }, + { + "name": "transformer.h.27.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1056768 + }, + { + "name": "transformer.h.27.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 1064960 + }, + { + "name": "transformer.h.27.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 23609344 + }, + { + "name": "transformer.h.27.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 26427392 + }, + { + "name": "transformer.h.28.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 32063488 + } + ], + "md5sum": "321b7ed8cb77889e57bf03271268e93d" + }, + { + "dataPath": "params_shard_104.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.28.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "c9dc93d765572b647df8be9876fbed1d" + }, + { + "dataPath": "params_shard_105.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.28.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "6798d2cc1093d6a6ada27ca5911c8943" + }, + { + "dataPath": "params_shard_106.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.29.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "8c1a4216bc753c81cd490735559d667d" + }, + { + "dataPath": "params_shard_107.bin", + "format": "raw-shard", + "nbytes": 32612352, + "records": [ + { + "name": "transformer.h.28.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 0 + }, + { + "name": "transformer.h.28.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 3145728 + }, + { + "name": "transformer.h.28.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 11534336 + }, + { + "name": "transformer.h.28.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.28.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12591104 + }, + { + "name": "transformer.h.28.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 12599296 + }, + { + "name": "transformer.h.28.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 15417344 + }, + { + "name": "transformer.h.29.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 21053440 + }, + { + "name": "transformer.h.29.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.29.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 24223744 + } + ], + "md5sum": "3949723acbb5dd23cf342072fe7c83fc" + }, + { + "dataPath": "params_shard_108.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.29.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "2f6568ebb3c3061d644cb7cc16f8278e" + }, + { + "dataPath": "params_shard_109.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.30.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "eabb17c5010feb10d97b864b6aa245c0" + }, + { + "dataPath": "params_shard_110.bin", + "format": "raw-shard", + "nbytes": 32088064, + "records": [ + { + "name": "transformer.h.29.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 0 + }, + { + "name": "transformer.h.29.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1048576 + }, + { + "name": "transformer.h.29.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1056768 + }, + { + "name": "transformer.h.29.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 1064960 + }, + { + "name": "transformer.h.29.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 23609344 + }, + { + "name": "transformer.h.29.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 26427392 + }, + { + "name": "transformer.h.30.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 32063488 + } + ], + "md5sum": "2838b343273c2e19f1f368cc886426ee" + }, + { + "dataPath": "params_shard_111.bin", + "format": "raw-shard", + "nbytes": 22544384, + "records": [ + { + "name": "transformer.h.30.mlp.c_proj.q_weight", + "shape": [ + 4096, + 1376 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 22544384, + "byteOffset": 0 + } + ], + "md5sum": "8df84bc0250ffb4891fbd827eee008c6" + }, + { + "dataPath": "params_shard_112.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.30.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "16b01ee4f39dd13c0c094d213f8f1803" + }, + { + "dataPath": "params_shard_113.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.31.attn.c_attn.q_weight", + "shape": [ + 12288, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "313b257a4960d5a048dc2f8c4ec42aa6" + }, + { + "dataPath": "params_shard_114.bin", + "format": "raw-shard", + "nbytes": 32612352, + "records": [ + { + "name": "transformer.h.30.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 0 + }, + { + "name": "transformer.h.30.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 3145728 + }, + { + "name": "transformer.h.30.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 11534336 + }, + { + "name": "transformer.h.30.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.30.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 12591104 + }, + { + "name": "transformer.h.30.mlp.c_proj.q_scale", + "shape": [ + 4096, + 344 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 2818048, + "byteOffset": 12599296 + }, + { + "name": "transformer.h.30.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 15417344 + }, + { + "name": "transformer.h.31.attn.c_attn.bias", + "shape": [ + 12288 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 24576, + "byteOffset": 21053440 + }, + { + "name": "transformer.h.31.attn.c_attn.q_scale", + "shape": [ + 12288, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 21078016 + }, + { + "name": "transformer.h.31.attn.c_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 24223744 + } + ], + "md5sum": "157c6a1a4b074583d4343d7500a4a474" + }, + { + "dataPath": "params_shard_115.bin", + "format": "raw-shard", + "nbytes": 45088768, + "records": [ + { + "name": "transformer.h.31.mlp.gate_up_proj.q_weight", + "shape": [ + 22016, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 45088768, + "byteOffset": 0 + } + ], + "md5sum": "c8417d3c04ebbebe607353d9ec312f86" + }, + { + "dataPath": "params_shard_116.bin", + "format": "raw-shard", + "nbytes": 6701056, + "records": [ + { + "name": "transformer.h.31.attn.c_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 0 + }, + { + "name": "transformer.h.31.ln_1.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1048576 + }, + { + "name": "transformer.h.31.ln_2.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 1056768 + }, + { + "name": "transformer.h.31.mlp.gate_up_proj.q_scale", + "shape": [ + 22016, + 128 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 5636096, + "byteOffset": 1064960 + } + ], + "md5sum": "34251c8f8a6dcd2ca8afe969cc77be27" + } + ] +} \ No newline at end of file