Views
No views yet
transformers version 4.39.1 or higher1# pip install 'transformers>=4.39.1'
2from transformers import AutoTokenizer, AutoModelForCausalLM
3
4model_id = "CohereForAI/c4ai-command-r-v01"
5tokenizer = AutoTokenizer.from_pretrained(model_id)
6model = AutoModelForCausalLM.from_pretrained(model_id)
7
8# Format message with the command-r chat template
9messages = [{"role": "user", "content": "Hello, how are you?"}]
10input_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt")
11## <BOS_TOKEN><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello, how are you?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
12
13gen_tokens = model.generate(
14 input_ids,
15 max_new_tokens=100,
16 do_sample=True,
17 temperature=0.3,
18 )
19
20gen_text = tokenizer.decode(gen_tokens[0])
21print(gen_text)1# pip install 'transformers>=4.39.1' bitsandbytes accelerate
2from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
3
4bnb_config = BitsAndBytesConfig(load_in_8bit=True)
5
6model_id = "CohereForAI/c4ai-command-r-v01"
7tokenizer = AutoTokenizer.from_pretrained(model_id)
8model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config)
9
10# Format message with the command-r chat template
11messages = [{"role": "user", "content": "Hello, how are you?"}]
12input_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt")
13## <BOS_TOKEN><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello, how are you?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
14
15gen_tokens = model.generate(
16 input_ids,
17 max_new_tokens=100,
18 do_sample=True,
19 temperature=0.3,
20 )
21
22gen_text = tokenizer.decode(gen_tokens[0])
23print(gen_text)