Add files using upload-large-folder tool

configuration_longcat_flash.py

@@ -0,0 +1,216 @@
+
+"""LongcatFlash model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.modeling_rope_utils import rope_config_validation
+
+
+LONGCAT_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
+
+
+class LongcatFlashConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`LongcatFlashModel`]. It is used to instantiate an LongcatFlash
+    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+    defaults will yield a similar configuration to that of the LongcatFlash.
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 131072):
+            Vocabulary size of the Deep model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`LongcatFlashModel`]
+        hidden_size (`int`, *optional*, defaults to 7168):
+            Dimension of the hidden representations.
+        ffn_hidden_size (`int`, *optional*, defaults to 18432):
+            Dimension of the MLP representations.
+        expert_ffn_hidden_size (`int`, *optional*, defaults to 2048):
+            Dimension of the MoE representations.
+        num_layers (`int`, *optional*, defaults to 61):
+            Number of hidden layers in the Transformer decoder.
+        num_attention_heads (`int`, *optional*, defaults to 128):
+            Number of attention heads for each attention layer in the Transformer decoder.
+        num_key_value_heads (`int`, *optional*, defaults to 128):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        n_routed_experts (`int`, *optional*, defaults to 256):
+            Number of routed experts.
+        routed_scaling_factor (`float`, *optional*, defaults to 2.5):
+            Scaling factor or routed experts.
+        kv_lora_rank (`int`, *optional*, defaults to 512):
+            Rank of the LoRA matrices for key and value projections.
+        q_lora_rank (`int`, *optional*, defaults to 1536):
+            Rank of the LoRA matrices for query projections.
+        qk_rope_head_dim (`int`, *optional*, defaults to 64):
+            Dimension of the query/key heads that use rotary position embeddings.
+        v_head_dim (`int`, *optional*, defaults to 128):
+            Dimension of the value heads.
+        qk_nope_head_dim (`int`, *optional*, defaults to 128):
+            Dimension of the query/key heads that don't use rotary position embeddings.
+        norm_topk_prob (`bool`, *optional*, defaults to `True`):
+            Whether to normalize the weights of the routed experts.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 4096):
+            The maximum sequence length that this model might ever be used with.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*):
+            Padding token id.
+        bos_token_id (`int`, *optional*, defaults to 0):
+            Beginning of stream token id.
+        eos_token_id (`int`, *optional*, defaults to 1):
+            End of stream token id.
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
+            Whether to use a bias in the query, key, value and output projection layers during self-attention.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        attention_method (`str`, *optional*, defaults to `"MLA"`):
+            The attention method to use.
+        initializer_range (`float`, *optional*, defaults to 0.006):
+            The initializer range for the model.
+        router_bias (`bool`, *optional*, defaults to `False`):
+            Whether to use a bias in the router.
+        zero_expert_num (`int`, *optional*, defaults to `None`):
+            The number of zero experts to use.
+        zero_expert_type (`str`, *optional*, defaults to `None`):
+            The type of zero expert to use.
+
+    ```python
+    >>> from transformers import LongcatFlashModel, LongcatFlashConfig
+
+    >>> # Initializing a LongcatFlash style configuration
+    >>> configuration = LongcatFlashConfig()
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "longcat_flash"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    base_model_tp_plan = {
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.experts.*.gate_proj": "local_colwise",
+        "layers.*.mlp.experts.*.up_proj": "local_colwise",
+        "layers.*.mlp.experts.*.down_proj": "local_rowwise",
+        "layers.*.mlps.*.gate_proj": "local_colwise",
+        "layers.*.mlps.*.up_proj": "local_colwise",
+        "layers.*.mlps.*.down_proj": "local_rowwise",
+    }
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+
+    def __init__(
+        self,
+        vocab_size=131072,
+        hidden_size=7168,
+        ffn_hidden_size=18432,
+        expert_ffn_hidden_size=2048,
+        num_layers=61,
+        num_attention_heads=128,
+        num_key_value_heads=None,
+        n_routed_experts=256,
+        routed_scaling_factor=1,
+        kv_lora_rank=512,
+        q_lora_rank=1536,
+        qk_rope_head_dim=64,
+        v_head_dim=128,
+        qk_nope_head_dim=128,
+        mla_scale_q_lora=True,
+        mla_scale_kv_lora=True,
+        moe_topk=8,
+        norm_topk_prob=False,
+        hidden_act="silu",
+        max_position_embeddings=4096,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=0,
+        eos_token_id=1,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        attention_bias=False,
+        attention_dropout=0.0,
+        attention_method='MLA',
+        initializer_range=0.006,
+        router_bias=False,
+        zero_expert_num=None,
+        zero_expert_type=None,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.ffn_hidden_size = ffn_hidden_size
+        self.expert_ffn_hidden_size = expert_ffn_hidden_size
+        self.num_layers = num_layers
+        self.num_attention_heads = num_attention_heads
+        self.n_routed_experts = n_routed_experts
+        self.routed_scaling_factor = routed_scaling_factor
+        self.kv_lora_rank = kv_lora_rank
+        self.q_lora_rank = q_lora_rank
+        self.qk_rope_head_dim = qk_rope_head_dim
+        self.v_head_dim = v_head_dim
+        self.qk_nope_head_dim = qk_nope_head_dim
+        self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
+        self.moe_topk = moe_topk
+        self.norm_topk_prob = norm_topk_prob
+        self.mla_scale_q_lora = mla_scale_q_lora
+        self.mla_scale_kv_lora = mla_scale_kv_lora
+        self.attention_method = attention_method
+        self.initializer_range = initializer_range
+        self.router_bias = router_bias
+        self.zero_expert_num = zero_expert_num
+        self.zero_expert_type = zero_expert_type
+
+        if self.attention_method == "MLA":
+            self.head_dim = qk_rope_head_dim
+        else:
+            ValueError('attention_method should be one of ["MLA"]')
+
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+
+        rope_config_validation(self)
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+    @property
+    def num_hidden_layers(self):
+        return self.num_layers
+
+
+__all__ = ["LongcatFlashConfig"]

model-00002-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6c3642a43834d84bdd4b340c19a783800f85980ad2024607d7264850d62b056
+size 12884902050

model-00003-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce899f54b6b938785051d263d88d29b11541db42c579b40955c83fb85cc94416
+size 12884902048

model-00006-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:213f50fb5eb44ed85ffff93dba3f234300644f00d1f2d9f5c9a21d8c17d29da9
+size 12884902050

model-00007-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e5a4b55b2bae8242125904a1094fee1a260b2454122c605617779cda3b67fa8
+size 12884902048

model-00008-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d37c34bb07abf1a947bf320ea94f459629f91912069ac0dbe13b019653192b92
+size 12884902050

model-00009-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fda5ecab837778e3a80805f417b2f0a5a2b91fc328f1a3c0d4db1cf37c5a497
+size 1287190563

model-00011-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ddd2c8bd3be49040f256125da2ad984f8caf7d656a38276ade670a5f29112e3
+size 12884902048

model-00012-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b018db0ae69cd481983efe97d033e2573f97f1723d54c82856b238f9767702c0
+size 12884902050

model-00014-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74d5be04dc2f8eef67577525a3f50168ecf95c677c0c6f43c93234c471c49450
+size 12884902050

model-00023-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab24c83982b9a9cd4c45880efc004e94a166727baeb3b3000ea5291c56b5995b
+size 12884902048

model-00026-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e3e3eea1a4878f35b53434e0f4e222dfa6a3f6beb29c64fc2badd362064de77
+size 12884902050

model-00029-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00c12e3b0ec899692bc71dc9e2794310ffc75ac8e91a257f8cd372459018d21e
+size 1287190571

model-00031-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf7109ffdf447724c32944e7003cd19c6a3b7e844ffba387731895c3b64db569
+size 12884902048

model-00032-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a42a8f82909205d00b59f6259fa260c11661b088eaff667efcf9caf115532b0
+size 12884902050

model-00034-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff21fd4cb2d4ed3ee1f0b757296ff2c4340f0f9b6cac2b6d7836533be8359cdc
+size 12884902050

model-00038-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0219428bcb5e1321b7c2300ec3b529f967d1c60ba1bb8ee5365e2b318d6afdf
+size 12884902050

model-00042-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:203d8c97ad9d0604d6008425e4b2ab3045a85e77d003237f4e4a17d338352a4b
+size 12884902051

model-00044-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4c51e3b26326657eac5655c49dffed9897657e0b892e67a34542ee7720e45be
+size 12884902051

model-00046-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86fde3754dc0e6a2e3d29d335b463d1be485d7053028c0cd732c1a228e37d5e2
+size 12884902051

model-00047-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49800421b65c53358cf6b18acaa584b7cc5615e0df63f52563f74bcef1327ec5
+size 12884902049

model-00048-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af00ddbceab437f69f76acb55f6a9e6001b0851087f7494a5580dc6c408fc06a
+size 12884902051

model-00049-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bda8c9ce328ca72565bbb98bf6e1fb67ffecae8c2d373500143570e813b0575d
+size 1287190597

model-00050-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81b5621042ae4ad59e32e8ee8b1b15886240ac18a711f3165b318716a2f6051b
+size 12884902051

model-00055-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e71fb1de346a3441a9560414d81d5929d71913d9e8b190c7fa65c65436fa98a
+size 12884902049

model-00056-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0633f1bc6af5c389f12ca6265ddc91d427bfbcc8224f335c7586d0b20b46e1d9
+size 12884902051

model-00059-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8e7c8fd08d707b5e355a4242eab2885fcd275e245f2414095c60ecb80221e1b
+size 12884902049

model-00060-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bf39fe6f4b766da028eebc22bc1b646feb11ff9e560c449bc90f61e61e55a7c
+size 12884902051

model-00062-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13c7ed633e24ec36ceef3b8dada55b1faefbff47da7da0d21e775dffd802335b
+size 12884902051

model-00064-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c5392e523c31bd4c4395fb3c9b731c075393908ae5272e21164d22375532e68
+size 12884902051

model-00067-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d02fede6144faf7c43e26bc956dec08c36fe8f1ff7ce595fa64cc18c7f74456
+size 12884902049

model-00068-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7357d5d80c3de714d4dd2991ce0e4449de45b521f837fa5fd1c73dcc5aa60b3d
+size 12884902051

model-00070-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5a69cfad5750ff4d895aaa5c2e02418259e0401d53f869bee2a8f6649fee5a1
+size 12884902051

model-00072-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bef1d87e5f6e90d76cfa7ac34a9ee53925b8596add97c05b18949673f71bd0f9
+size 12884902051

model-00075-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9b02a84d347c06001a2cb9fb8f851bbdf65eb64cab374568cc494f0058e1385
+size 12884902049

model-00076-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba090ebc267beccbea6b8cb853b9a175b460cf68f245485385bff0be8fd8dcad
+size 12884902051

model-00079-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bc8873de7fa7d09c4dce08aa4d8ade20d7b270466bf98130af94b2921280bad
+size 12884902049

model-00080-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5746666d9350279d1321fa434d4fae445354c56015db1c5f46d163d3176f65a6
+size 12884902051

model-00083-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:016c47119b339ce64d1abe16ffe2aa9dcf6e6003159f6d5b4812eeed4451ac37
+size 12884902049

model-00086-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c84ff641df1a3d371e9c674d49017fb93edfb806c1e6677e5fa4b2b87dbd8e6
+size 12884902051

model-00088-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b983e979fc9cfdd201f6a8259ea694396767cda2d1d9614a06d7f5972bbb6762
+size 12884902051

model-00089-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40dee467454160e565468845220aa98304fd63ece02ec2bf6e616eaea3dae885
+size 1287190609

model-00091-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02cf08a286523031e29b17eeb758ee4c796a9c320a365d35f9d4ea649ce0b996
+size 12884902049

model-00094-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee6c35fa1bf91295266cc099cb3eb0d889878310ae9d4c2f167f60dc06bdd0ab
+size 12884902051

model-00098-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e4d118c4ae7b05a155bd0f497bd42801b8249dc087612deaa5cd370b283b099
+size 12884902051

model-00102-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b0c78078538fb7f490651a23a059fca1203f1b52bfe2dc68c70245fe7cc89b8
+size 12884902051

model-00107-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6ef77ab298df729e1d110d11500494170301e62ec819f8a8e50e2c1e9ae084d
+size 12884902049

model-00108-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e8d60abf6d39a3fc8916bd463e4f3c9439c32778667a54b6dc34f71f2d69f6e
+size 12884902051

model-00110-of-00113.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94c5e90c0e3a55290af9c2e31af73e5c3a665b35c4c6461c9d7a83954e681d60
+size 12884902051

model.safetensors.index.json

@@ -0,0 +1,823 @@
+{
+    "metadata": {
+        "total_size": 1121594245120,
+        "total_parameters": 560664958976
+    },
+    "weight_map": {
+        "lm_head.weight": "model-00113-of-00113.safetensors",
+        "model.embed_tokens.weight": "model-00001-of-00113.safetensors",
+        "model.layers.0.input_layernorm.0.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.input_layernorm.1.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.mlp.router.classifier.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.mlp.router.e_score_correction_bias": "model-00005-of-00113.safetensors",
+        "model.layers.0.mlp.switch_mlp.down_proj.weight": "model-00004-of-00113.safetensors",
+        "model.layers.0.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00113.safetensors",
+        "model.layers.0.mlp.switch_mlp.up_proj.weight": "model-00003-of-00113.safetensors",
+        "model.layers.0.mlps.0.down_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.mlps.0.gate_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.mlps.0.up_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.mlps.1.down_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.mlps.1.gate_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.mlps.1.up_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.post_attention_layernorm.0.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.post_attention_layernorm.1.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.0.kv_a_layernorm.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.0.kv_a_proj_with_mqa.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.0.kv_b_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.0.o_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.0.q_a_layernorm.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.0.q_a_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.0.q_b_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.1.kv_a_layernorm.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.1.kv_a_proj_with_mqa.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.1.kv_b_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.1.o_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.1.q_a_layernorm.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.1.q_a_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.0.self_attn.1.q_b_proj.weight": "model-00005-of-00113.safetensors",
+        "model.layers.1.input_layernorm.0.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.input_layernorm.1.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.mlp.router.classifier.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.mlp.router.e_score_correction_bias": "model-00009-of-00113.safetensors",
+        "model.layers.1.mlp.switch_mlp.down_proj.weight": "model-00008-of-00113.safetensors",
+        "model.layers.1.mlp.switch_mlp.gate_proj.weight": "model-00006-of-00113.safetensors",
+        "model.layers.1.mlp.switch_mlp.up_proj.weight": "model-00007-of-00113.safetensors",
+        "model.layers.1.mlps.0.down_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.mlps.0.gate_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.mlps.0.up_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.mlps.1.down_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.mlps.1.gate_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.mlps.1.up_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.post_attention_layernorm.0.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.post_attention_layernorm.1.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.0.kv_a_layernorm.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.0.kv_a_proj_with_mqa.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.0.kv_b_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.0.o_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.0.q_a_layernorm.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.0.q_a_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.0.q_b_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.1.kv_a_layernorm.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.1.kv_a_proj_with_mqa.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.1.kv_b_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.1.o_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.1.q_a_layernorm.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.1.q_a_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.1.self_attn.1.q_b_proj.weight": "model-00009-of-00113.safetensors",
+        "model.layers.10.input_layernorm.0.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.input_layernorm.1.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.mlp.router.classifier.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.mlp.router.e_score_correction_bias": "model-00045-of-00113.safetensors",
+        "model.layers.10.mlp.switch_mlp.down_proj.weight": "model-00044-of-00113.safetensors",
+        "model.layers.10.mlp.switch_mlp.gate_proj.weight": "model-00042-of-00113.safetensors",
+        "model.layers.10.mlp.switch_mlp.up_proj.weight": "model-00043-of-00113.safetensors",
+        "model.layers.10.mlps.0.down_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.mlps.0.gate_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.mlps.0.up_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.mlps.1.down_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.mlps.1.gate_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.mlps.1.up_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.post_attention_layernorm.0.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.post_attention_layernorm.1.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.0.kv_a_layernorm.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.0.kv_a_proj_with_mqa.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.0.kv_b_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.0.o_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.0.q_a_layernorm.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.0.q_a_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.0.q_b_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.1.kv_a_layernorm.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.1.kv_a_proj_with_mqa.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.1.kv_b_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.1.o_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.1.q_a_layernorm.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.1.q_a_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.10.self_attn.1.q_b_proj.weight": "model-00045-of-00113.safetensors",
+        "model.layers.11.input_layernorm.0.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.input_layernorm.1.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.mlp.router.classifier.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.mlp.router.e_score_correction_bias": "model-00049-of-00113.safetensors",
+        "model.layers.11.mlp.switch_mlp.down_proj.weight": "model-00048-of-00113.safetensors",
+        "model.layers.11.mlp.switch_mlp.gate_proj.weight": "model-00046-of-00113.safetensors",
+        "model.layers.11.mlp.switch_mlp.up_proj.weight": "model-00047-of-00113.safetensors",
+        "model.layers.11.mlps.0.down_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.mlps.0.gate_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.mlps.0.up_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.mlps.1.down_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.mlps.1.gate_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.mlps.1.up_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.post_attention_layernorm.0.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.post_attention_layernorm.1.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.0.kv_a_layernorm.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.0.kv_a_proj_with_mqa.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.0.kv_b_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.0.o_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.0.q_a_layernorm.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.0.q_a_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.0.q_b_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.1.kv_a_layernorm.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.1.kv_a_proj_with_mqa.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.1.kv_b_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.1.o_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.1.q_a_layernorm.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.1.q_a_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.11.self_attn.1.q_b_proj.weight": "model-00049-of-00113.safetensors",
+        "model.layers.12.input_layernorm.0.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.input_layernorm.1.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.mlp.router.classifier.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.mlp.router.e_score_correction_bias": "model-00053-of-00113.safetensors",
+        "model.layers.12.mlp.switch_mlp.down_proj.weight": "model-00052-of-00113.safetensors",
+        "model.layers.12.mlp.switch_mlp.gate_proj.weight": "model-00050-of-00113.safetensors",
+        "model.layers.12.mlp.switch_mlp.up_proj.weight": "model-00051-of-00113.safetensors",
+        "model.layers.12.mlps.0.down_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.mlps.0.gate_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.mlps.0.up_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.mlps.1.down_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.mlps.1.gate_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.mlps.1.up_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.post_attention_layernorm.0.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.post_attention_layernorm.1.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.0.kv_a_layernorm.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.0.kv_a_proj_with_mqa.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.0.kv_b_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.0.o_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.0.q_a_layernorm.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.0.q_a_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.0.q_b_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.1.kv_a_layernorm.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.1.kv_a_proj_with_mqa.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.1.kv_b_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.1.o_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.1.q_a_layernorm.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.1.q_a_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.12.self_attn.1.q_b_proj.weight": "model-00053-of-00113.safetensors",
+        "model.layers.13.input_layernorm.0.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.input_layernorm.1.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.mlp.router.classifier.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.mlp.router.e_score_correction_bias": "model-00057-of-00113.safetensors",
+        "model.layers.13.mlp.switch_mlp.down_proj.weight": "model-00056-of-00113.safetensors",
+        "model.layers.13.mlp.switch_mlp.gate_proj.weight": "model-00054-of-00113.safetensors",
+        "model.layers.13.mlp.switch_mlp.up_proj.weight": "model-00055-of-00113.safetensors",
+        "model.layers.13.mlps.0.down_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.mlps.0.gate_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.mlps.0.up_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.mlps.1.down_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.mlps.1.gate_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.mlps.1.up_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.post_attention_layernorm.0.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.post_attention_layernorm.1.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.0.kv_a_layernorm.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.0.kv_a_proj_with_mqa.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.0.kv_b_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.0.o_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.0.q_a_layernorm.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.0.q_a_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.0.q_b_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.1.kv_a_layernorm.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.1.kv_a_proj_with_mqa.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.1.kv_b_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.1.o_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.1.q_a_layernorm.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.1.q_a_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.13.self_attn.1.q_b_proj.weight": "model-00057-of-00113.safetensors",
+        "model.layers.14.input_layernorm.0.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.input_layernorm.1.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.mlp.router.classifier.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.mlp.router.e_score_correction_bias": "model-00061-of-00113.safetensors",
+        "model.layers.14.mlp.switch_mlp.down_proj.weight": "model-00060-of-00113.safetensors",
+        "model.layers.14.mlp.switch_mlp.gate_proj.weight": "model-00058-of-00113.safetensors",
+        "model.layers.14.mlp.switch_mlp.up_proj.weight": "model-00059-of-00113.safetensors",
+        "model.layers.14.mlps.0.down_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.mlps.0.gate_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.mlps.0.up_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.mlps.1.down_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.mlps.1.gate_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.mlps.1.up_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.post_attention_layernorm.0.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.post_attention_layernorm.1.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.0.kv_a_layernorm.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.0.kv_a_proj_with_mqa.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.0.kv_b_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.0.o_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.0.q_a_layernorm.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.0.q_a_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.0.q_b_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.1.kv_a_layernorm.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.1.kv_a_proj_with_mqa.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.1.kv_b_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.1.o_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.1.q_a_layernorm.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.1.q_a_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.14.self_attn.1.q_b_proj.weight": "model-00061-of-00113.safetensors",
+        "model.layers.15.input_layernorm.0.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.input_layernorm.1.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.mlp.router.classifier.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.mlp.router.e_score_correction_bias": "model-00065-of-00113.safetensors",
+        "model.layers.15.mlp.switch_mlp.down_proj.weight": "model-00064-of-00113.safetensors",
+        "model.layers.15.mlp.switch_mlp.gate_proj.weight": "model-00062-of-00113.safetensors",
+        "model.layers.15.mlp.switch_mlp.up_proj.weight": "model-00063-of-00113.safetensors",
+        "model.layers.15.mlps.0.down_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.mlps.0.gate_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.mlps.0.up_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.mlps.1.down_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.mlps.1.gate_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.mlps.1.up_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.post_attention_layernorm.0.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.post_attention_layernorm.1.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.0.kv_a_layernorm.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.0.kv_a_proj_with_mqa.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.0.kv_b_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.0.o_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.0.q_a_layernorm.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.0.q_a_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.0.q_b_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.1.kv_a_layernorm.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.1.kv_a_proj_with_mqa.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.1.kv_b_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.1.o_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.1.q_a_layernorm.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.1.q_a_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.15.self_attn.1.q_b_proj.weight": "model-00065-of-00113.safetensors",
+        "model.layers.16.input_layernorm.0.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.input_layernorm.1.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.mlp.router.classifier.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.mlp.router.e_score_correction_bias": "model-00069-of-00113.safetensors",
+        "model.layers.16.mlp.switch_mlp.down_proj.weight": "model-00068-of-00113.safetensors",
+        "model.layers.16.mlp.switch_mlp.gate_proj.weight": "model-00066-of-00113.safetensors",
+        "model.layers.16.mlp.switch_mlp.up_proj.weight": "model-00067-of-00113.safetensors",
+        "model.layers.16.mlps.0.down_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.mlps.0.gate_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.mlps.0.up_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.mlps.1.down_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.mlps.1.gate_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.mlps.1.up_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.post_attention_layernorm.0.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.post_attention_layernorm.1.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.0.kv_a_layernorm.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.0.kv_a_proj_with_mqa.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.0.kv_b_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.0.o_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.0.q_a_layernorm.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.0.q_a_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.0.q_b_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.1.kv_a_layernorm.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.1.kv_a_proj_with_mqa.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.1.kv_b_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.1.o_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.1.q_a_layernorm.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.1.q_a_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.16.self_attn.1.q_b_proj.weight": "model-00069-of-00113.safetensors",
+        "model.layers.17.input_layernorm.0.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.input_layernorm.1.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.mlp.router.classifier.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.mlp.router.e_score_correction_bias": "model-00073-of-00113.safetensors",
+        "model.layers.17.mlp.switch_mlp.down_proj.weight": "model-00072-of-00113.safetensors",
+        "model.layers.17.mlp.switch_mlp.gate_proj.weight": "model-00070-of-00113.safetensors",
+        "model.layers.17.mlp.switch_mlp.up_proj.weight": "model-00071-of-00113.safetensors",
+        "model.layers.17.mlps.0.down_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.mlps.0.gate_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.mlps.0.up_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.mlps.1.down_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.mlps.1.gate_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.mlps.1.up_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.post_attention_layernorm.0.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.post_attention_layernorm.1.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.0.kv_a_layernorm.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.0.kv_a_proj_with_mqa.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.0.kv_b_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.0.o_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.0.q_a_layernorm.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.0.q_a_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.0.q_b_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.1.kv_a_layernorm.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.1.kv_a_proj_with_mqa.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.1.kv_b_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.1.o_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.1.q_a_layernorm.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.1.q_a_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.17.self_attn.1.q_b_proj.weight": "model-00073-of-00113.safetensors",
+        "model.layers.18.input_layernorm.0.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.input_layernorm.1.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.mlp.router.classifier.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.mlp.router.e_score_correction_bias": "model-00077-of-00113.safetensors",
+        "model.layers.18.mlp.switch_mlp.down_proj.weight": "model-00076-of-00113.safetensors",
+        "model.layers.18.mlp.switch_mlp.gate_proj.weight": "model-00074-of-00113.safetensors",
+        "model.layers.18.mlp.switch_mlp.up_proj.weight": "model-00075-of-00113.safetensors",
+        "model.layers.18.mlps.0.down_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.mlps.0.gate_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.mlps.0.up_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.mlps.1.down_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.mlps.1.gate_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.mlps.1.up_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.post_attention_layernorm.0.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.post_attention_layernorm.1.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.0.kv_a_layernorm.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.0.kv_a_proj_with_mqa.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.0.kv_b_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.0.o_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.0.q_a_layernorm.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.0.q_a_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.0.q_b_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.1.kv_a_layernorm.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.1.kv_a_proj_with_mqa.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.1.kv_b_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.1.o_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.1.q_a_layernorm.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.1.q_a_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.18.self_attn.1.q_b_proj.weight": "model-00077-of-00113.safetensors",
+        "model.layers.19.input_layernorm.0.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.input_layernorm.1.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.mlp.router.classifier.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.mlp.router.e_score_correction_bias": "model-00081-of-00113.safetensors",
+        "model.layers.19.mlp.switch_mlp.down_proj.weight": "model-00080-of-00113.safetensors",
+        "model.layers.19.mlp.switch_mlp.gate_proj.weight": "model-00078-of-00113.safetensors",
+        "model.layers.19.mlp.switch_mlp.up_proj.weight": "model-00079-of-00113.safetensors",
+        "model.layers.19.mlps.0.down_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.mlps.0.gate_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.mlps.0.up_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.mlps.1.down_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.mlps.1.gate_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.mlps.1.up_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.post_attention_layernorm.0.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.post_attention_layernorm.1.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.0.kv_a_layernorm.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.0.kv_a_proj_with_mqa.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.0.kv_b_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.0.o_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.0.q_a_layernorm.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.0.q_a_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.0.q_b_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.1.kv_a_layernorm.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.1.kv_a_proj_with_mqa.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.1.kv_b_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.1.o_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.1.q_a_layernorm.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.1.q_a_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.19.self_attn.1.q_b_proj.weight": "model-00081-of-00113.safetensors",
+        "model.layers.2.input_layernorm.0.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.input_layernorm.1.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.mlp.router.classifier.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.mlp.router.e_score_correction_bias": "model-00013-of-00113.safetensors",
+        "model.layers.2.mlp.switch_mlp.down_proj.weight": "model-00012-of-00113.safetensors",
+        "model.layers.2.mlp.switch_mlp.gate_proj.weight": "model-00010-of-00113.safetensors",
+        "model.layers.2.mlp.switch_mlp.up_proj.weight": "model-00011-of-00113.safetensors",
+        "model.layers.2.mlps.0.down_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.mlps.0.gate_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.mlps.0.up_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.mlps.1.down_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.mlps.1.gate_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.mlps.1.up_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.post_attention_layernorm.0.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.post_attention_layernorm.1.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.0.kv_a_layernorm.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.0.kv_a_proj_with_mqa.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.0.kv_b_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.0.o_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.0.q_a_layernorm.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.0.q_a_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.0.q_b_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.1.kv_a_layernorm.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.1.kv_a_proj_with_mqa.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.1.kv_b_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.1.o_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.1.q_a_layernorm.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.1.q_a_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.2.self_attn.1.q_b_proj.weight": "model-00013-of-00113.safetensors",
+        "model.layers.20.input_layernorm.0.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.input_layernorm.1.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.mlp.router.classifier.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.mlp.router.e_score_correction_bias": "model-00085-of-00113.safetensors",
+        "model.layers.20.mlp.switch_mlp.down_proj.weight": "model-00084-of-00113.safetensors",
+        "model.layers.20.mlp.switch_mlp.gate_proj.weight": "model-00082-of-00113.safetensors",
+        "model.layers.20.mlp.switch_mlp.up_proj.weight": "model-00083-of-00113.safetensors",
+        "model.layers.20.mlps.0.down_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.mlps.0.gate_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.mlps.0.up_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.mlps.1.down_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.mlps.1.gate_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.mlps.1.up_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.post_attention_layernorm.0.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.post_attention_layernorm.1.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.0.kv_a_layernorm.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.0.kv_a_proj_with_mqa.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.0.kv_b_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.0.o_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.0.q_a_layernorm.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.0.q_a_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.0.q_b_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.1.kv_a_layernorm.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.1.kv_a_proj_with_mqa.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.1.kv_b_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.1.o_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.1.q_a_layernorm.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.1.q_a_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.20.self_attn.1.q_b_proj.weight": "model-00085-of-00113.safetensors",
+        "model.layers.21.input_layernorm.0.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.input_layernorm.1.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.mlp.router.classifier.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.mlp.router.e_score_correction_bias": "model-00089-of-00113.safetensors",
+        "model.layers.21.mlp.switch_mlp.down_proj.weight": "model-00088-of-00113.safetensors",
+        "model.layers.21.mlp.switch_mlp.gate_proj.weight": "model-00086-of-00113.safetensors",
+        "model.layers.21.mlp.switch_mlp.up_proj.weight": "model-00087-of-00113.safetensors",
+        "model.layers.21.mlps.0.down_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.mlps.0.gate_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.mlps.0.up_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.mlps.1.down_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.mlps.1.gate_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.mlps.1.up_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.post_attention_layernorm.0.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.post_attention_layernorm.1.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.0.kv_a_layernorm.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.0.kv_a_proj_with_mqa.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.0.kv_b_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.0.o_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.0.q_a_layernorm.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.0.q_a_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.0.q_b_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.1.kv_a_layernorm.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.1.kv_a_proj_with_mqa.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.1.kv_b_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.1.o_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.1.q_a_layernorm.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.1.q_a_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.21.self_attn.1.q_b_proj.weight": "model-00089-of-00113.safetensors",
+        "model.layers.22.input_layernorm.0.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.input_layernorm.1.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.mlp.router.classifier.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.mlp.router.e_score_correction_bias": "model-00093-of-00113.safetensors",
+        "model.layers.22.mlp.switch_mlp.down_proj.weight": "model-00092-of-00113.safetensors",
+        "model.layers.22.mlp.switch_mlp.gate_proj.weight": "model-00090-of-00113.safetensors",
+        "model.layers.22.mlp.switch_mlp.up_proj.weight": "model-00091-of-00113.safetensors",
+        "model.layers.22.mlps.0.down_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.mlps.0.gate_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.mlps.0.up_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.mlps.1.down_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.mlps.1.gate_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.mlps.1.up_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.post_attention_layernorm.0.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.post_attention_layernorm.1.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.0.kv_a_layernorm.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.0.kv_a_proj_with_mqa.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.0.kv_b_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.0.o_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.0.q_a_layernorm.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.0.q_a_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.0.q_b_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.1.kv_a_layernorm.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.1.kv_a_proj_with_mqa.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.1.kv_b_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.1.o_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.1.q_a_layernorm.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.1.q_a_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.22.self_attn.1.q_b_proj.weight": "model-00093-of-00113.safetensors",
+        "model.layers.23.input_layernorm.0.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.input_layernorm.1.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.mlp.router.classifier.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.mlp.router.e_score_correction_bias": "model-00097-of-00113.safetensors",
+        "model.layers.23.mlp.switch_mlp.down_proj.weight": "model-00096-of-00113.safetensors",
+        "model.layers.23.mlp.switch_mlp.gate_proj.weight": "model-00094-of-00113.safetensors",
+        "model.layers.23.mlp.switch_mlp.up_proj.weight": "model-00095-of-00113.safetensors",
+        "model.layers.23.mlps.0.down_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.mlps.0.gate_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.mlps.0.up_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.mlps.1.down_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.mlps.1.gate_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.mlps.1.up_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.post_attention_layernorm.0.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.post_attention_layernorm.1.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.0.kv_a_layernorm.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.0.kv_a_proj_with_mqa.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.0.kv_b_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.0.o_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.0.q_a_layernorm.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.0.q_a_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.0.q_b_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.1.kv_a_layernorm.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.1.kv_a_proj_with_mqa.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.1.kv_b_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.1.o_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.1.q_a_layernorm.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.1.q_a_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.23.self_attn.1.q_b_proj.weight": "model-00097-of-00113.safetensors",
+        "model.layers.24.input_layernorm.0.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.input_layernorm.1.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.mlp.router.classifier.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.mlp.router.e_score_correction_bias": "model-00101-of-00113.safetensors",
+        "model.layers.24.mlp.switch_mlp.down_proj.weight": "model-00100-of-00113.safetensors",
+        "model.layers.24.mlp.switch_mlp.gate_proj.weight": "model-00098-of-00113.safetensors",
+        "model.layers.24.mlp.switch_mlp.up_proj.weight": "model-00099-of-00113.safetensors",
+        "model.layers.24.mlps.0.down_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.mlps.0.gate_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.mlps.0.up_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.mlps.1.down_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.mlps.1.gate_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.mlps.1.up_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.post_attention_layernorm.0.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.post_attention_layernorm.1.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.0.kv_a_layernorm.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.0.kv_a_proj_with_mqa.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.0.kv_b_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.0.o_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.0.q_a_layernorm.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.0.q_a_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.0.q_b_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.1.kv_a_layernorm.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.1.kv_a_proj_with_mqa.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.1.kv_b_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.1.o_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.1.q_a_layernorm.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.1.q_a_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.24.self_attn.1.q_b_proj.weight": "model-00101-of-00113.safetensors",
+        "model.layers.25.input_layernorm.0.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.input_layernorm.1.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.mlp.router.classifier.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.mlp.router.e_score_correction_bias": "model-00105-of-00113.safetensors",
+        "model.layers.25.mlp.switch_mlp.down_proj.weight": "model-00104-of-00113.safetensors",
+        "model.layers.25.mlp.switch_mlp.gate_proj.weight": "model-00102-of-00113.safetensors",
+        "model.layers.25.mlp.switch_mlp.up_proj.weight": "model-00103-of-00113.safetensors",
+        "model.layers.25.mlps.0.down_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.mlps.0.gate_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.mlps.0.up_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.mlps.1.down_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.mlps.1.gate_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.mlps.1.up_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.post_attention_layernorm.0.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.post_attention_layernorm.1.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.0.kv_a_layernorm.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.0.kv_a_proj_with_mqa.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.0.kv_b_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.0.o_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.0.q_a_layernorm.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.0.q_a_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.0.q_b_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.1.kv_a_layernorm.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.1.kv_a_proj_with_mqa.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.1.kv_b_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.1.o_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.1.q_a_layernorm.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.1.q_a_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.25.self_attn.1.q_b_proj.weight": "model-00105-of-00113.safetensors",
+        "model.layers.26.input_layernorm.0.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.input_layernorm.1.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.mlp.router.classifier.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.mlp.router.e_score_correction_bias": "model-00109-of-00113.safetensors",
+        "model.layers.26.mlp.switch_mlp.down_proj.weight": "model-00108-of-00113.safetensors",
+        "model.layers.26.mlp.switch_mlp.gate_proj.weight": "model-00106-of-00113.safetensors",
+        "model.layers.26.mlp.switch_mlp.up_proj.weight": "model-00107-of-00113.safetensors",
+        "model.layers.26.mlps.0.down_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.mlps.0.gate_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.mlps.0.up_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.mlps.1.down_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.mlps.1.gate_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.mlps.1.up_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.post_attention_layernorm.0.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.post_attention_layernorm.1.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.0.kv_a_layernorm.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.0.kv_a_proj_with_mqa.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.0.kv_b_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.0.o_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.0.q_a_layernorm.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.0.q_a_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.0.q_b_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.1.kv_a_layernorm.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.1.kv_a_proj_with_mqa.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.1.kv_b_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.1.o_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.1.q_a_layernorm.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.1.q_a_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.26.self_attn.1.q_b_proj.weight": "model-00109-of-00113.safetensors",
+        "model.layers.27.input_layernorm.0.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.input_layernorm.1.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.mlp.router.classifier.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.mlp.router.e_score_correction_bias": "model-00113-of-00113.safetensors",
+        "model.layers.27.mlp.switch_mlp.down_proj.weight": "model-00112-of-00113.safetensors",
+        "model.layers.27.mlp.switch_mlp.gate_proj.weight": "model-00110-of-00113.safetensors",
+        "model.layers.27.mlp.switch_mlp.up_proj.weight": "model-00111-of-00113.safetensors",
+        "model.layers.27.mlps.0.down_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.mlps.0.gate_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.mlps.0.up_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.mlps.1.down_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.mlps.1.gate_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.mlps.1.up_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.post_attention_layernorm.0.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.post_attention_layernorm.1.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.0.kv_a_layernorm.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.0.kv_a_proj_with_mqa.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.0.kv_b_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.0.o_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.0.q_a_layernorm.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.0.q_a_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.0.q_b_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.1.kv_a_layernorm.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.1.kv_a_proj_with_mqa.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.1.kv_b_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.1.o_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.1.q_a_layernorm.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.1.q_a_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.27.self_attn.1.q_b_proj.weight": "model-00113-of-00113.safetensors",
+        "model.layers.3.input_layernorm.0.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.input_layernorm.1.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.mlp.router.classifier.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.mlp.router.e_score_correction_bias": "model-00017-of-00113.safetensors",
+        "model.layers.3.mlp.switch_mlp.down_proj.weight": "model-00016-of-00113.safetensors",
+        "model.layers.3.mlp.switch_mlp.gate_proj.weight": "model-00014-of-00113.safetensors",
+        "model.layers.3.mlp.switch_mlp.up_proj.weight": "model-00015-of-00113.safetensors",
+        "model.layers.3.mlps.0.down_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.mlps.0.gate_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.mlps.0.up_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.mlps.1.down_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.mlps.1.gate_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.mlps.1.up_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.post_attention_layernorm.0.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.post_attention_layernorm.1.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.0.kv_a_layernorm.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.0.kv_a_proj_with_mqa.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.0.kv_b_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.0.o_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.0.q_a_layernorm.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.0.q_a_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.0.q_b_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.1.kv_a_layernorm.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.1.kv_a_proj_with_mqa.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.1.kv_b_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.1.o_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.1.q_a_layernorm.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.1.q_a_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.3.self_attn.1.q_b_proj.weight": "model-00017-of-00113.safetensors",
+        "model.layers.4.input_layernorm.0.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.input_layernorm.1.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.mlp.router.classifier.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.mlp.router.e_score_correction_bias": "model-00021-of-00113.safetensors",
+        "model.layers.4.mlp.switch_mlp.down_proj.weight": "model-00020-of-00113.safetensors",
+        "model.layers.4.mlp.switch_mlp.gate_proj.weight": "model-00018-of-00113.safetensors",
+        "model.layers.4.mlp.switch_mlp.up_proj.weight": "model-00019-of-00113.safetensors",
+        "model.layers.4.mlps.0.down_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.mlps.0.gate_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.mlps.0.up_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.mlps.1.down_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.mlps.1.gate_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.mlps.1.up_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.post_attention_layernorm.0.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.post_attention_layernorm.1.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.0.kv_a_layernorm.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.0.kv_a_proj_with_mqa.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.0.kv_b_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.0.o_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.0.q_a_layernorm.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.0.q_a_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.0.q_b_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.1.kv_a_layernorm.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.1.kv_a_proj_with_mqa.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.1.kv_b_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.1.o_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.1.q_a_layernorm.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.1.q_a_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.4.self_attn.1.q_b_proj.weight": "model-00021-of-00113.safetensors",
+        "model.layers.5.input_layernorm.0.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.input_layernorm.1.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.mlp.router.classifier.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.mlp.router.e_score_correction_bias": "model-00025-of-00113.safetensors",
+        "model.layers.5.mlp.switch_mlp.down_proj.weight": "model-00024-of-00113.safetensors",
+        "model.layers.5.mlp.switch_mlp.gate_proj.weight": "model-00022-of-00113.safetensors",
+        "model.layers.5.mlp.switch_mlp.up_proj.weight": "model-00023-of-00113.safetensors",
+        "model.layers.5.mlps.0.down_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.mlps.0.gate_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.mlps.0.up_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.mlps.1.down_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.mlps.1.gate_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.mlps.1.up_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.post_attention_layernorm.0.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.post_attention_layernorm.1.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.0.kv_a_layernorm.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.0.kv_a_proj_with_mqa.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.0.kv_b_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.0.o_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.0.q_a_layernorm.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.0.q_a_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.0.q_b_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.1.kv_a_layernorm.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.1.kv_a_proj_with_mqa.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.1.kv_b_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.1.o_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.1.q_a_layernorm.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.1.q_a_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.5.self_attn.1.q_b_proj.weight": "model-00025-of-00113.safetensors",
+        "model.layers.6.input_layernorm.0.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.input_layernorm.1.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.mlp.router.classifier.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.mlp.router.e_score_correction_bias": "model-00029-of-00113.safetensors",
+        "model.layers.6.mlp.switch_mlp.down_proj.weight": "model-00028-of-00113.safetensors",
+        "model.layers.6.mlp.switch_mlp.gate_proj.weight": "model-00026-of-00113.safetensors",
+        "model.layers.6.mlp.switch_mlp.up_proj.weight": "model-00027-of-00113.safetensors",
+        "model.layers.6.mlps.0.down_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.mlps.0.gate_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.mlps.0.up_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.mlps.1.down_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.mlps.1.gate_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.mlps.1.up_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.post_attention_layernorm.0.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.post_attention_layernorm.1.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.0.kv_a_layernorm.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.0.kv_a_proj_with_mqa.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.0.kv_b_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.0.o_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.0.q_a_layernorm.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.0.q_a_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.0.q_b_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.1.kv_a_layernorm.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.1.kv_a_proj_with_mqa.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.1.kv_b_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.1.o_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.1.q_a_layernorm.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.1.q_a_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.6.self_attn.1.q_b_proj.weight": "model-00029-of-00113.safetensors",
+        "model.layers.7.input_layernorm.0.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.input_layernorm.1.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.mlp.router.classifier.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.mlp.router.e_score_correction_bias": "model-00033-of-00113.safetensors",
+        "model.layers.7.mlp.switch_mlp.down_proj.weight": "model-00032-of-00113.safetensors",
+        "model.layers.7.mlp.switch_mlp.gate_proj.weight": "model-00030-of-00113.safetensors",
+        "model.layers.7.mlp.switch_mlp.up_proj.weight": "model-00031-of-00113.safetensors",
+        "model.layers.7.mlps.0.down_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.mlps.0.gate_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.mlps.0.up_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.mlps.1.down_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.mlps.1.gate_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.mlps.1.up_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.post_attention_layernorm.0.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.post_attention_layernorm.1.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.0.kv_a_layernorm.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.0.kv_a_proj_with_mqa.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.0.kv_b_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.0.o_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.0.q_a_layernorm.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.0.q_a_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.0.q_b_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.1.kv_a_layernorm.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.1.kv_a_proj_with_mqa.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.1.kv_b_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.1.o_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.1.q_a_layernorm.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.1.q_a_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.7.self_attn.1.q_b_proj.weight": "model-00033-of-00113.safetensors",
+        "model.layers.8.input_layernorm.0.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.input_layernorm.1.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.mlp.router.classifier.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.mlp.router.e_score_correction_bias": "model-00037-of-00113.safetensors",
+        "model.layers.8.mlp.switch_mlp.down_proj.weight": "model-00036-of-00113.safetensors",
+        "model.layers.8.mlp.switch_mlp.gate_proj.weight": "model-00034-of-00113.safetensors",
+        "model.layers.8.mlp.switch_mlp.up_proj.weight": "model-00035-of-00113.safetensors",
+        "model.layers.8.mlps.0.down_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.mlps.0.gate_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.mlps.0.up_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.mlps.1.down_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.mlps.1.gate_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.mlps.1.up_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.post_attention_layernorm.0.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.post_attention_layernorm.1.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.0.kv_a_layernorm.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.0.kv_a_proj_with_mqa.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.0.kv_b_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.0.o_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.0.q_a_layernorm.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.0.q_a_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.0.q_b_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.1.kv_a_layernorm.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.1.kv_a_proj_with_mqa.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.1.kv_b_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.1.o_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.1.q_a_layernorm.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.1.q_a_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.8.self_attn.1.q_b_proj.weight": "model-00037-of-00113.safetensors",
+        "model.layers.9.input_layernorm.0.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.input_layernorm.1.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.mlp.router.classifier.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.mlp.router.e_score_correction_bias": "model-00041-of-00113.safetensors",
+        "model.layers.9.mlp.switch_mlp.down_proj.weight": "model-00040-of-00113.safetensors",
+        "model.layers.9.mlp.switch_mlp.gate_proj.weight": "model-00038-of-00113.safetensors",
+        "model.layers.9.mlp.switch_mlp.up_proj.weight": "model-00039-of-00113.safetensors",
+        "model.layers.9.mlps.0.down_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.mlps.0.gate_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.mlps.0.up_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.mlps.1.down_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.mlps.1.gate_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.mlps.1.up_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.post_attention_layernorm.0.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.post_attention_layernorm.1.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.0.kv_a_layernorm.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.0.kv_a_proj_with_mqa.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.0.kv_b_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.0.o_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.0.q_a_layernorm.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.0.q_a_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.0.q_b_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.1.kv_a_layernorm.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.1.kv_a_proj_with_mqa.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.1.kv_b_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.1.o_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.1.q_a_layernorm.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.1.q_a_proj.weight": "model-00041-of-00113.safetensors",
+        "model.layers.9.self_attn.1.q_b_proj.weight": "model-00041-of-00113.safetensors",
+        "model.norm.weight": "model-00113-of-00113.safetensors"
+    }
+}