manaestras commited on
Commit
51c1a22
·
verified ·
1 Parent(s): 987eba7

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +5 -0
tokenizer_config.json CHANGED
@@ -6550,6 +6550,11 @@
6550
  "eos_token": "<|hy_Assistant|>",
6551
  "model_max_length": 1000000000000000019884624838656,
6552
  "pad_token": "<|hy_▁pad▁|>",
 
 
 
 
 
6553
  "tokenizer_class": "PreTrainedTokenizerFast",
6554
  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% if messages[0]['content'] is string %}{% set system_message = messages[0]['content'] %}{% else %}{% set system_message = messages[0]['content']['text'] %}{% endif %}<|hy_begin▁of▁sentence|>{{ system_message }}<|hy_place▁holder▁no▁3|>{% else %}{% set loop_messages = messages %}<|hy_begin▁of▁sentence|>{% endif %}{% for message in loop_messages %}{% if message['role'] == 'user' %}{% if message['content'] is string %}{{ message['content'] }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}<|hy_place▁holder▁no▁100|><|hy_place▁holder▁no▁102|><|hy_place▁holder▁no▁101|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}<|hy_User|>{% elif message['role'] == 'assistant' %}{{ message['content'] }}<|hy_Assistant|>{% endif %}{% endfor %}"
6555
  }
 
6550
  "eos_token": "<|hy_Assistant|>",
6551
  "model_max_length": 1000000000000000019884624838656,
6552
  "pad_token": "<|hy_▁pad▁|>",
6553
+ "extra_sepcial_tokens": {
6554
+ "image_token": "<|hy_place▁holder▁no▁102|>",
6555
+ "image_start_token": "<|hy_place▁holder▁no▁100|>",
6556
+ "image_end_token": "<|hy_place▁holder▁no▁101|>"
6557
+ },
6558
  "tokenizer_class": "PreTrainedTokenizerFast",
6559
  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% if messages[0]['content'] is string %}{% set system_message = messages[0]['content'] %}{% else %}{% set system_message = messages[0]['content']['text'] %}{% endif %}<|hy_begin▁of▁sentence|>{{ system_message }}<|hy_place▁holder▁no▁3|>{% else %}{% set loop_messages = messages %}<|hy_begin▁of▁sentence|>{% endif %}{% for message in loop_messages %}{% if message['role'] == 'user' %}{% if message['content'] is string %}{{ message['content'] }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}<|hy_place▁holder▁no▁100|><|hy_place▁holder▁no▁102|><|hy_place▁holder▁no▁101|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}<|hy_User|>{% elif message['role'] == 'assistant' %}{{ message['content'] }}<|hy_Assistant|>{% endif %}{% endfor %}"
6560
  }