feat(transformers): Add native Transformers config

config.json

@@ -21,7 +21,7 @@
   "layer_norm_eps": 1e-12,
   "layer_norm_type": "layer_norm",
   "max_position_embeddings": 8192,
-  "model_type": "new",
+  "model_type": "gte",
   "num_attention_heads": 12,
   "num_hidden_layers": 12,
   "pack_qkv": true,
@@ -34,5 +34,9 @@
   "type_vocab_size": 0,
   "unpad_inputs": false,
   "use_memory_efficient_attention": false,
-  "vocab_size": 30528
+  "vocab_size": 30528,
+  "rope_parameters": {
+    "rope_theta": 500000,
+    "rope_type": "default"
+  }
 }