Text Generation
Transformers
Safetensors
chatglm
feature-extraction
custom_code
JosephusCheung commited on
Commit
dc1056a
·
1 Parent(s): 9f55c75

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -1
tokenizer_config.json CHANGED
@@ -123,7 +123,8 @@
123
  "<|user|>", "<|assistant|>", "<|observation|>", "<|begin_of_image|>", "<|end_of_image|>",
124
  "<|begin_of_video|>", "<|end_of_video|>"],
125
  "clean_up_tokenization_spaces": false,
126
- "chat_template": "[gMASK]<sop>{% for item in messages %}{% if item['content'] %}<|{{ item['role'] }}|>{{ item['metadata'] }}\n{{ item['content'] }}{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>\n{% endif %}", "do_lower_case": false,
 
127
  "eos_token": "<|endoftext|>",
128
  "pad_token": "<|endoftext|>",
129
  "model_max_length": 1024000,
 
123
  "<|user|>", "<|assistant|>", "<|observation|>", "<|begin_of_image|>", "<|end_of_image|>",
124
  "<|begin_of_video|>", "<|end_of_video|>"],
125
  "clean_up_tokenization_spaces": false,
126
+ "chat_template": "[gMASK]<sop>{% for item in messages %}{% if item['content'] %}<|{{ item['role'] }}|>{{ item['metadata'] }}\n{{ item['content'] }}{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>\n{% endif %}",
127
+ "do_lower_case": false,
128
  "eos_token": "<|endoftext|>",
129
  "pad_token": "<|endoftext|>",
130
  "model_max_length": 1024000,