Hugging Face's logo

aws-neuron
/
optimum-neuron-cache

Model card Files Community
239
dacorvo HF staff commited on
Commit
20e585f
•
1 Parent(s): d30345f

Add Mistral-v2

Browse files
Files changed (1) hide show
  1. inference-cache-config/mistral.json +50 -0
inference-cache-config/mistral.json CHANGED
@@ -1,4 +1,48 @@
1
  {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  "mistralai/Mistral-7B-Instruct-v0.1": [
3
  {
4
  "batch_size": 1,
@@ -35,6 +79,12 @@
35
  "sequence_length": 4096,
36
  "num_cores": 8,
37
  "auto_cast_type": "bf16"
 
 
 
 
 
 
38
  }
39
  ]
40
  }
 
1
  {
2
+ "mistralai/Mistral-7B-Instruct-v0.2": [
3
+ {
4
+ "batch_size": 1,
5
+ "sequence_length": 4096,
6
+ "num_cores": 2,
7
+ "auto_cast_type": "bf16"
8
+ },
9
+ {
10
+ "batch_size": 1,
11
+ "sequence_length": 4096,
12
+ "num_cores": 8,
13
+ "auto_cast_type": "bf16"
14
+ },
15
+ {
16
+ "batch_size": 4,
17
+ "sequence_length": 4096,
18
+ "num_cores": 2,
19
+ "auto_cast_type": "bf16"
20
+ },
21
+ {
22
+ "batch_size": 4,
23
+ "sequence_length": 4096,
24
+ "num_cores": 8,
25
+ "auto_cast_type": "bf16"
26
+ },
27
+ {
28
+ "batch_size": 8,
29
+ "sequence_length": 4096,
30
+ "num_cores": 8,
31
+ "auto_cast_type": "bf16"
32
+ },
33
+ {
34
+ "batch_size": 16,
35
+ "sequence_length": 4096,
36
+ "num_cores": 8,
37
+ "auto_cast_type": "bf16"
38
+ },
39
+ {
40
+ "batch_size": 32,
41
+ "sequence_length": 4096,
42
+ "num_cores": 8,
43
+ "auto_cast_type": "bf16"
44
+ }
45
+ ],
46
  "mistralai/Mistral-7B-Instruct-v0.1": [
47
  {
48
  "batch_size": 1,
 
79
  "sequence_length": 4096,
80
  "num_cores": 8,
81
  "auto_cast_type": "bf16"
82
+ },
83
+ {
84
+ "batch_size": 32,
85
+ "sequence_length": 4096,
86
+ "num_cores": 8,
87
+ "auto_cast_type": "bf16"
88
  }
89
  ]
90
  }