Views
No views yet
1from vllm import LLM
2from vllm.sampling_params import SamplingParams
3
4
5model_name = "akahana/indo-psikologi-sft"
6sampling_params = SamplingParams(max_tokens=256, temperature=0.0)
7llm = LLM(
8 model=model_name,
9 trust_remote_code=True,
10 max_model_len=2048,
11 max_num_batched_tokens=2048,
12 tensor_parallel_size=2, # if you have multigpus
13)
14
15prompt = "### psikologi ### \n Hai aku lagi terpuruk hari ini"
16
17messages = [
18 {
19 "role": "user",
20 "content": prompt,
21 }
22]
23
24outputs = llm.chat(messages, sampling_params=sampling_params)
25output = outputs[0].outputs[0].text.strip()
26messages.append({"role":"assistant","content":output})
27output