Views
No views yet
1import torch
2from transformers import AutoTokenizer, AutoModelForCausalLM
3
4model_id = "retrieva-jp/Llama-3-Swallow-8B-Instruct-v0.1-kokoroe"
5tokenizer = AutoTokenizer.from_pretrained(model_id)
6model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype=torch.bfloat16)
7chat = [
8 {"role": "system", "content": "あなたは誠実で優秀な日本人のアシスタントです。"},
9 {"role": "user", "content": "自然言語処理とは何か"},
10]
11tokenized_input = tokenizer.apply_chat_template(chat, add_generation_prompt=True, tokenize=True, return_tensors="pt").to(model.device)
12with torch.no_grad():
13 output = model.generate(
14 tokenized_input,
15 max_new_tokens=100,
16 do_sample=True,
17 top_p=0.9,
18 temperature=0.6,
19 )[0]
20print(tokenizer.decode(output))1$ vllm serve --model retrieva-jp/Llama-3-Swallow-8B-Instruct-v0.1-kokoroe --port 8000
2$ curl -X POST http://localhost:8000/generate \
3 -H "Content-Type: application/json" \
4 -d '{
5 "prompt": [
6 {"role": "system", "content": "あなたは誠実で優秀な日本人のアシスタントです。"},
7 {"role": "user", "content": "自然言語処理とは何か"}
8 ],
9 "max_new_tokens": 100,
10 "do_sample": true,
11 "top_p": 0.9,
12 "temperature": 0.6
13 }'