TomLucidor commited on
Commit
82cc165
·
verified ·
1 Parent(s): 69dd744

Upload config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. config.json +237 -0
config.json ADDED
@@ -0,0 +1,237 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "BailingMoeV2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_bailing_moe_v2.BailingMoeV2Config",
8
+ "AutoModel": "modeling_bailing_moe_v2.BailingMoeV2Model",
9
+ "AutoModelForCausalLM": "modeling_bailing_moe_v2.BailingMoeV2ForCausalLM"
10
+ },
11
+ "embedding_dropout": 0.0,
12
+ "eos_token_id": 156892,
13
+ "first_k_dense_replace": 1,
14
+ "head_dim": 128,
15
+ "hidden_act": "silu",
16
+ "hidden_size": 2048,
17
+ "intermediate_size": 5120,
18
+ "max_position_embeddings": 32768,
19
+ "moba_block_size": 1024,
20
+ "moba_layer_freq": [
21
+ 0,
22
+ 0,
23
+ 0,
24
+ 1,
25
+ 1,
26
+ 1,
27
+ 1,
28
+ 1,
29
+ 1,
30
+ 1,
31
+ 1,
32
+ 1,
33
+ 1,
34
+ 1,
35
+ 1,
36
+ 1,
37
+ 1,
38
+ 1,
39
+ 1,
40
+ 0
41
+ ],
42
+ "moba_topk": 8,
43
+ "model_type": "bailing_moe",
44
+ "moe_intermediate_size": 512,
45
+ "moe_router_enable_expert_bias": true,
46
+ "moe_shared_expert_intermediate_size": 512,
47
+ "n_group": 8,
48
+ "norm_topk_prob": true,
49
+ "num_attention_heads": 16,
50
+ "num_experts": 256,
51
+ "num_experts_per_tok": 8,
52
+ "num_hidden_layers": 20,
53
+ "num_key_value_heads": 4,
54
+ "num_shared_experts": 1,
55
+ "output_dropout": 0.0,
56
+ "pad_token_id": 156892,
57
+ "partial_rotary_factor": 0.5,
58
+ "quantization": {
59
+ "group_size": 64,
60
+ "bits": 4,
61
+ "mode": "affine",
62
+ "model.layers.1.mlp.gate.gate_proj": {
63
+ "group_size": 64,
64
+ "bits": 8
65
+ },
66
+ "model.layers.2.mlp.gate.gate_proj": {
67
+ "group_size": 64,
68
+ "bits": 8
69
+ },
70
+ "model.layers.3.mlp.gate.gate_proj": {
71
+ "group_size": 64,
72
+ "bits": 8
73
+ },
74
+ "model.layers.4.mlp.gate.gate_proj": {
75
+ "group_size": 64,
76
+ "bits": 8
77
+ },
78
+ "model.layers.5.mlp.gate.gate_proj": {
79
+ "group_size": 64,
80
+ "bits": 8
81
+ },
82
+ "model.layers.6.mlp.gate.gate_proj": {
83
+ "group_size": 64,
84
+ "bits": 8
85
+ },
86
+ "model.layers.7.mlp.gate.gate_proj": {
87
+ "group_size": 64,
88
+ "bits": 8
89
+ },
90
+ "model.layers.8.mlp.gate.gate_proj": {
91
+ "group_size": 64,
92
+ "bits": 8
93
+ },
94
+ "model.layers.9.mlp.gate.gate_proj": {
95
+ "group_size": 64,
96
+ "bits": 8
97
+ },
98
+ "model.layers.10.mlp.gate.gate_proj": {
99
+ "group_size": 64,
100
+ "bits": 8
101
+ },
102
+ "model.layers.11.mlp.gate.gate_proj": {
103
+ "group_size": 64,
104
+ "bits": 8
105
+ },
106
+ "model.layers.12.mlp.gate.gate_proj": {
107
+ "group_size": 64,
108
+ "bits": 8
109
+ },
110
+ "model.layers.13.mlp.gate.gate_proj": {
111
+ "group_size": 64,
112
+ "bits": 8
113
+ },
114
+ "model.layers.14.mlp.gate.gate_proj": {
115
+ "group_size": 64,
116
+ "bits": 8
117
+ },
118
+ "model.layers.15.mlp.gate.gate_proj": {
119
+ "group_size": 64,
120
+ "bits": 8
121
+ },
122
+ "model.layers.16.mlp.gate.gate_proj": {
123
+ "group_size": 64,
124
+ "bits": 8
125
+ },
126
+ "model.layers.17.mlp.gate.gate_proj": {
127
+ "group_size": 64,
128
+ "bits": 8
129
+ },
130
+ "model.layers.18.mlp.gate.gate_proj": {
131
+ "group_size": 64,
132
+ "bits": 8
133
+ },
134
+ "model.layers.19.mlp.gate.gate_proj": {
135
+ "group_size": 64,
136
+ "bits": 8
137
+ }
138
+ },
139
+ "quantization_config": {
140
+ "group_size": 64,
141
+ "bits": 4,
142
+ "mode": "affine",
143
+ "model.layers.1.mlp.gate.gate_proj": {
144
+ "group_size": 64,
145
+ "bits": 8
146
+ },
147
+ "model.layers.2.mlp.gate.gate_proj": {
148
+ "group_size": 64,
149
+ "bits": 8
150
+ },
151
+ "model.layers.3.mlp.gate.gate_proj": {
152
+ "group_size": 64,
153
+ "bits": 8
154
+ },
155
+ "model.layers.4.mlp.gate.gate_proj": {
156
+ "group_size": 64,
157
+ "bits": 8
158
+ },
159
+ "model.layers.5.mlp.gate.gate_proj": {
160
+ "group_size": 64,
161
+ "bits": 8
162
+ },
163
+ "model.layers.6.mlp.gate.gate_proj": {
164
+ "group_size": 64,
165
+ "bits": 8
166
+ },
167
+ "model.layers.7.mlp.gate.gate_proj": {
168
+ "group_size": 64,
169
+ "bits": 8
170
+ },
171
+ "model.layers.8.mlp.gate.gate_proj": {
172
+ "group_size": 64,
173
+ "bits": 8
174
+ },
175
+ "model.layers.9.mlp.gate.gate_proj": {
176
+ "group_size": 64,
177
+ "bits": 8
178
+ },
179
+ "model.layers.10.mlp.gate.gate_proj": {
180
+ "group_size": 64,
181
+ "bits": 8
182
+ },
183
+ "model.layers.11.mlp.gate.gate_proj": {
184
+ "group_size": 64,
185
+ "bits": 8
186
+ },
187
+ "model.layers.12.mlp.gate.gate_proj": {
188
+ "group_size": 64,
189
+ "bits": 8
190
+ },
191
+ "model.layers.13.mlp.gate.gate_proj": {
192
+ "group_size": 64,
193
+ "bits": 8
194
+ },
195
+ "model.layers.14.mlp.gate.gate_proj": {
196
+ "group_size": 64,
197
+ "bits": 8
198
+ },
199
+ "model.layers.15.mlp.gate.gate_proj": {
200
+ "group_size": 64,
201
+ "bits": 8
202
+ },
203
+ "model.layers.16.mlp.gate.gate_proj": {
204
+ "group_size": 64,
205
+ "bits": 8
206
+ },
207
+ "model.layers.17.mlp.gate.gate_proj": {
208
+ "group_size": 64,
209
+ "bits": 8
210
+ },
211
+ "model.layers.18.mlp.gate.gate_proj": {
212
+ "group_size": 64,
213
+ "bits": 8
214
+ },
215
+ "model.layers.19.mlp.gate.gate_proj": {
216
+ "group_size": 64,
217
+ "bits": 8
218
+ }
219
+ },
220
+ "rms_norm_eps": 1e-06,
221
+ "rope_scaling": null,
222
+ "rope_theta": 600000,
223
+ "routed_scaling_factor": 2.5,
224
+ "router_dtype": "fp32",
225
+ "score_function": "sigmoid",
226
+ "tie_word_embeddings": false,
227
+ "topk_group": 4,
228
+ "torch_dtype": "bfloat16",
229
+ "transformers_version": "4.52.3",
230
+ "use_bias": false,
231
+ "use_cache": true,
232
+ "use_moba_decode": true,
233
+ "use_qk_norm": true,
234
+ "use_qkv_bias": false,
235
+ "use_rmsnorm": true,
236
+ "vocab_size": 157184
237
+ }