Views
No views yet
1text = tokenizer.apply_chat_template([
2 {"role" : "system", "content" : SYSTEM_PROMPT},
3 {"role" : "user", "content" : "Different Cyber law in India?"},
4], tokenize = False, add_generation_prompt = True)
5
6from vllm import SamplingParams
7sampling_params = SamplingParams(
8 temperature = 0.8,
9 top_p = 0.95,
10 max_tokens = 1024,
11)
12output = model.fast_generate(
13 text,
14 sampling_params = sampling_params,
15 lora_request = model.load_lora("grpo_saved_lora"),
16)[0].outputs[0].text