Views
No views yet
1# Assuming you're using llama.cpp server
2import requests
3
4def translate(text: str, source_lang: str = "en", target_lang: str = "zh-CN") -> str:
5 prompt = (
6 f'user\n'
7 f'[{{"type": "text", "source_lang_code": "{source_lang}", "target_lang_code": "{target_lang}", "text": "{text}"}}]\n'
8 f'model\n'
9 )
10
11 result = ""
12 with requests.post(
13 "http://127.0.0.1:8080/v1/completions",
14 json={
15 "prompt": prompt,
16 "temperature": 0.1,
17 "stop": ["\nuser", "<eos>", "<end_of_turn>"],
18 "stream": True,
19 "max_tokens": 1024,
20 },
21 stream=True
22 ) as resp:
23 resp.raise_for_status()
24 for line in resp.iter_lines():
25 if not line:
26 continue
27 line = line.decode("utf-8")
28 if not line.startswith("data: "):
29 continue
30 data = json.loads(line[len("data: "):])
31 token = data["choices"][0]["text"]
32 print(token, end="", flush=True)
33 result += token
34 if data["choices"][0].get("finish_reason"):
35 break
36 print() # final newline
37 return result1@misc{vonwerra2022trl,
2 title = {{TRL: Transformer Reinforcement Learning}},
3 author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
4 year = 2020,
5 journal = {GitHub repository},
6 publisher = {GitHub},
7 howpublished = {\url{https://github.com/huggingface/trl}}
8}