Hugging Face's logo

aws-neuron
/
optimum-neuron-cache

Model card Files Community
311
dacorvo HF staff commited on
Commit
4e25bb0
·
verified ·
1 Parent(s): 38fdd8c

Add qwen2.5 config for models up to 14B params

Browse files
Files changed (1) hide show
  1. inference-cache-config/qwen2.5 +76 -0
inference-cache-config/qwen2.5 ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "Qwen/Qwen2.5-7B-Instruct": [
3
+ {
4
+ "batch_size": 1,
5
+ "sequence_length": 4096,
6
+ "num_cores": 2,
7
+ "auto_cast_type": "bf16"
8
+ },
9
+ {
10
+ "batch_size": 4,
11
+ "sequence_length": 4096,
12
+ "num_cores": 2,
13
+ "auto_cast_type": "bf16"
14
+ },
15
+ {
16
+ "batch_size": 8,
17
+ "sequence_length": 4096,
18
+ "num_cores": 2,
19
+ "auto_cast_type": "bf16"
20
+ },
21
+ {
22
+ "batch_size": 1,
23
+ "sequence_length": 4096,
24
+ "num_cores": 8,
25
+ "auto_cast_type": "bf16"
26
+ },
27
+ {
28
+ "batch_size": 32,
29
+ "sequence_length": 4096,
30
+ "num_cores": 8,
31
+ "auto_cast_type": "bf16"
32
+ }
33
+ ],
34
+ "Qwen/Qwen2.5-0.5B": [
35
+ {
36
+ "batch_size": 1,
37
+ "sequence_length": 4096,
38
+ "num_cores": 2,
39
+ "auto_cast_type": "bf16"
40
+ },
41
+ {
42
+ "batch_size": 4,
43
+ "sequence_length": 4096,
44
+ "num_cores": 2,
45
+ "auto_cast_type": "bf16"
46
+ }
47
+ ],
48
+ "Qwen/Qwen2.5-1.5B": [
49
+ {
50
+ "batch_size": 1,
51
+ "sequence_length": 4096,
52
+ "num_cores": 2,
53
+ "auto_cast_type": "bf16"
54
+ },
55
+ {
56
+ "batch_size": 4,
57
+ "sequence_length": 4096,
58
+ "num_cores": 2,
59
+ "auto_cast_type": "bf16"
60
+ }
61
+ ],
62
+ "Qwen/Qwen2.5-14B": [
63
+ {
64
+ "batch_size": 1,
65
+ "sequence_length": 4096,
66
+ "num_cores": 8,
67
+ "auto_cast_type": "bf16"
68
+ },
69
+ {
70
+ "batch_size": 16,
71
+ "sequence_length": 4096,
72
+ "num_cores": 8,
73
+ "auto_cast_type": "bf16"
74
+ }
75
+ ]
76
+ }