Views
No views yet
1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer
3
4model_name = "abeja/Mixtral-8x7B-Instruct-v0.1-japanese-alpha"
5
6tokenizer = AutoTokenizer.from_pretrained(model_name)
7model = AutoModelForCausalLM.from_pretrained(
8 model_name,
9 torch_dtype=torch.float16,
10 use_cache=True,
11 device_map="auto",
12)
13model.eval()
14
15input_text = """# system
16誠実で紳士的で優秀なAIアシスタントとして、簡潔でわかりやすく役に立つ回答を自信をもって答えなさい。
17
18# question
19人とAIが協調するためには？
20
21# answer"""
22input_ids = tokenizer.encode(input_text, return_tensors="pt")
23
24with torch.no_grad():
25 output_ids = model.generate(
26 input_ids.to(model.device),
27 max_new_tokens=256,
28 pad_token_id=tokenizer.pad_token_id,
29 eos_token_id=tokenizer.eos_token_id,
30 )
31output = tokenizer.decode(output_ids.tolist()[0], skip_special_tokens=True)
32print(output)