This repo contains GGUF format model which is a quantization of the model:
https://huggingface.co/ngoantech/Llama-2-7b-vietnamese-20k1from langchain.llms import LlamaCpp
2from langchain.prompts import PromptTemplate
3from langchain.chains import LLMChain
4from langchain.callbacks.manager import CallbackManager
5from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
6
7template = """Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.
8
9User: Chào Bob.
10Bob: Chào bạn. Tôi có thể giúp gì cho bạn?
11User: Thủ đô của Việt Nam là thành phố nào?
12Bob: Hà Nội là thủ đô của Việt Nam
13User: {question}"""
14
15# template = """<<SYS>>\nYou are a helpful assistant. Bạn là một trợ lí hữu ích.\n<</SYS>>\n\n[INST] {question} [/INST] """
16
17# template = """[INST] <<SYS>>
18# You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.
19# <</SYS>>
20
21# {question} [/INST]
22# """
23
24prompt = PromptTemplate(template=template, input_variables=["question"])
25
26# Callbacks support token-wise streaming
27callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
28
29# Make sure the model path is correct for your system!
30llm = LlamaCpp(
31 model_path="/path/to/model/gguf-model-q4_0.bin",
32 temperature=0.1,
33 max_tokens=1024,
34 top_p=1,
35 callback_manager=callback_manager,
36 verbose=True, # Verbose is required to pass to the callback manager
37)
38
39llm_chain = LLMChain(prompt=prompt, llm=llm)
40question = "Quốc khánh của Việt Nam diễn ra vào ngày nào?"
41print(prompt.format(question=question))
42llm_chain.run(question)
1git clone https://github.com/ggerganov/llama.cpp.git
2cd llama.cpp/ && make -j
3./main -m /path/to/model/gguf-model-q4_0.bin --temp 0.1 -t 8 -n 1024 --color -p "VNG Corporation là công ty công nghệ hàng đầu "
4./main -m /path/to/model/gguf-model-q4_0.bin --temp 0.1 -t 8 -n 1024 --color -r "User:" -f /path/to/chat/prompt/chat.txt
