Views
No views yet
1import torch
2from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
3
4model_id = "matsuo-lab/weblab-10b-instruction-sft"
5repo_id = "indiebot-community/weblab-10b-instruction-sft-bnb-4bit"
6
7bnb_config = BitsAndBytesConfig(
8 load_in_4bit=True,
9 bnb_4bit_use_double_quant=True,
10 bnb_4bit_quant_type="nf4",
11 bnb_4bit_compute_dtype=torch.bfloat16
12)
13
14tokenizer = AutoTokenizer.from_pretrained(model_id)
15model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config, device_map="auto")
16
17tokenizer.push_to_hub(repo_id)
18model.push_to_hub(repo_id)1{
2 (省略...)
3
4 "bos_token": "<|endoftext|>",
5 "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
6 "clean_up_tokenization_spaces": true,
7 "eos_token": "<|endoftext|>",
8 "extra_special_tokens": {},
9 "model_max_length": 1000000000000000019884624838656,
10 "pad_token": "<|padding|>",
11 "tokenizer_class": "PreTrainedTokenizer",
12 "unk_token": "<|endoftext|>"
13}