Views
No views yet
| Name | Quant method | Size |
|---|---|---|
| THaLLE-0.1-7B-fa.Q2_K.gguf | Q2_K | 2.81GB |
| THaLLE-0.1-7B-fa.IQ3_XS.gguf | IQ3_XS | 3.12GB |
| THaLLE-0.1-7B-fa.IQ3_S.gguf | IQ3_S | 3.26GB |
| THaLLE-0.1-7B-fa.Q3_K_S.gguf | Q3_K_S | 3.25GB |
| THaLLE-0.1-7B-fa.IQ3_M.gguf | IQ3_M | 3.33GB |
| THaLLE-0.1-7B-fa.Q3_K.gguf | Q3_K | 3.55GB |
| THaLLE-0.1-7B-fa.Q3_K_M.gguf | Q3_K_M | 3.55GB |
| THaLLE-0.1-7B-fa.Q3_K_L.gguf | Q3_K_L | 3.81GB |
| THaLLE-0.1-7B-fa.IQ4_XS.gguf | IQ4_XS | 3.96GB |
| THaLLE-0.1-7B-fa.Q4_0.gguf | Q4_0 | 4.13GB |
| THaLLE-0.1-7B-fa.IQ4_NL.gguf | IQ4_NL | 4.16GB |
| THaLLE-0.1-7B-fa.Q4_K_S.gguf | Q4_K_S | 4.15GB |
| THaLLE-0.1-7B-fa.Q4_K.gguf | Q4_K | 4.36GB |
| THaLLE-0.1-7B-fa.Q4_K_M.gguf | Q4_K_M | 4.36GB |
| THaLLE-0.1-7B-fa.Q4_1.gguf | Q4_1 | 4.54GB |
| THaLLE-0.1-7B-fa.Q5_0.gguf | Q5_0 | 4.95GB |
| THaLLE-0.1-7B-fa.Q5_K_S.gguf | Q5_K_S | 4.95GB |
| THaLLE-0.1-7B-fa.Q5_K.gguf | Q5_K | 5.07GB |
| THaLLE-0.1-7B-fa.Q5_K_M.gguf | Q5_K_M | 5.07GB |
| THaLLE-0.1-7B-fa.Q5_1.gguf | Q5_1 | 5.36GB |
| THaLLE-0.1-7B-fa.Q6_K.gguf | Q6_K | 5.82GB |
| THaLLE-0.1-7B-fa.Q8_0.gguf | Q8_0 | 7.54GB |
KBTG-Labs/THaLLE-0.1-7B-fa is a WIP model checkpoint distributed for reproducing results in our Technical Report.tokenizer_config.json bos_token field from null to the start token "<|im_start|>".1{
2 ...
3 "bos_token": "<|im_start|>"
4 ...
5}| Model | Internal 2020 | Internal 2024 | Flare CFA* |
|---|---|---|---|
| APIs | |||
gpt-3.5-turbo-0125 | 0.5458 | 0.5027 | 0.6366 |
gemini-1.5-flash-001 | 0.6271 | 0.6278 | 0.7355 |
gemini-1.5-pro-001 | 0.6780 | 0.6444 | 0.7829 |
gpt-4o-2024-05-13 | 0.8000 | 0.8055 | 0.8789 |
| HF models | |||
"meta-llama/Llama-2-7b-chat-hf" | 0.3774 | 0.3639 | 0.4264 |
"google/gemma-7b-it" | 0.5107 | 0.5333 | 0.6027 |
"meta-llama/Meta-Llama-3-8B-Instruct" | 0.5424 | 0.5222 | 0.6386 |
"Qwen/Qwen2-7B-Instruct" | 0.5740 | 0.5583 | 0.6831 |
"KBTG-Labs/THaLLE-0.1-7B-fa" | 0.6678 | 0.6500 | 0.7171 |
"ChanceFocus/flare-cfa"KBTG-Labs/THaLLE-0.1-7B-fa is a fine-tuned of Qwen2-7B-Instruct you will need to install transformers>=4.37.0.Progress: 1032/1032 | Correct: 740 (71.71%)1import re
2from typing import Literal, Optional
3
4import torch
5from datasets import load_dataset
6from transformers import AutoModelForCausalLM, AutoTokenizer
7
8MODEL_ID: str = "KBTG-Labs/THaLLE-0.1-7B-fa"
9SYSTEM_PROMPT: str = """You are a CFA (chartered financial analyst) taking a test to evaluate your knowledge of finance. You will be given a question along with three possible answers (A, B, and C).
10Indicate the correct answer (A, B, or C)."""
11QUESTION_TEMPLATE: str = """Question:
12{question}
13A. {choice_a}
14B. {choice_b}
15C. {choice_c}"""
16
17
18def format_flare_cfa(text: str) -> dict[str, str]:
19 text = re.sub(r"\s+", " ", text)
20
21 pattern = r"Q:\s*(.*?),\s*CHOICES:\s*A:\s*(.*?),\s*B:\s*(.*?),\s*C:\s*(.*)"
22 match = re.search(pattern, text)
23 if match:
24 question, choice_a, choice_b, choice_c = match.groups()
25 return {
26 "question": question.strip(),
27 "choice_a": choice_a.strip(),
28 "choice_b": choice_b.strip(),
29 "choice_c": choice_c.strip(),
30 }
31 else:
32 raise ValueError("Input text does not match the expected format.")
33
34
35def load_benchmark_dataset() -> list[dict[str, str]]:
36 dataset = load_dataset("ChanceFocus/flare-cfa")["test"]
37 prepared_dataset = []
38 for d in dataset:
39 entry = format_flare_cfa(d["text"])
40 entry["answer"] = str(d["answer"]).upper()
41 prepared_dataset.append(entry)
42 return prepared_dataset
43
44
45def extract_choice(
46 response_text: str, choice_a: str, choice_b: str, choice_c: str
47) -> Optional[Literal["A", "B", "C"]]:
48 def clean(text: str) -> str:
49 return text.replace("–", "-").strip().replace("\n", "")
50
51 find_choice = re.findall(
52 r"([T|t]he correct answer is[.|:]? [ABC]|[A|a]nswer[.|:]?[is]?\W+?\n?[ABC]\s)",
53 response_text,
54 )
55
56 if find_choice:
57 return clean(find_choice[0])[-1]
58
59 if len(response_text) == 1 and response_text in "ABC":
60 return response_text
61
62 find_choice = re.findall(r"[ABC][.]\s?", response_text)
63 if find_choice:
64 return find_choice[0][0]
65
66 choice = {"A": choice_a, "B": choice_b, "C": choice_c}
67
68 for ch, content in choice.items():
69 if clean(content) in clean(response_text):
70 return ch
71
72 return None
73
74
75def inference(messages: list[dict[str, str]], model, tokenizer) -> str:
76 text = tokenizer.apply_chat_template(
77 messages,
78 tokenize=False,
79 add_generation_prompt=True,
80 )
81 model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
82
83 generated_ids = model.generate(
84 model_inputs.input_ids,
85 max_new_tokens=768,
86 do_sample=False,
87 temperature=None,
88 top_p=None,
89 top_k=None,
90 )
91 generated_ids = [
92 output_ids[len(input_ids) :]
93 for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
94 ]
95
96 response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
97 return response
98
99
100def run_benchmark(dataset: list[dict[str, str]], model, tokenizer):
101 total_correct = 0
102
103 for i, problem in enumerate(dataset, start=1):
104 messages = [
105 {"role": "system", "content": SYSTEM_PROMPT},
106 {"role": "user", "content": QUESTION_TEMPLATE.format(**problem)},
107 ]
108 output_text = inference(messages, model, tokenizer)
109 prediction = extract_choice(
110 output_text,
111 problem["choice_a"],
112 problem["choice_b"],
113 problem["choice_c"],
114 )
115
116 correct = problem["answer"] == prediction
117 total_correct += correct
118 percent = total_correct / i * 100
119
120 print(
121 f"Progress: {i}/{len(dataset)} | Correct: {total_correct} ({percent:.2f}%)",
122 end="\r",
123 )
124
125
126if __name__ == "__main__":
127 dataset = load_benchmark_dataset()
128 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
129 model = AutoModelForCausalLM.from_pretrained(
130 MODEL_ID,
131 torch_dtype=torch.bfloat16,
132 device_map="auto",
133 )
134
135 run_benchmark(dataset, model, tokenizer)
136@misc{labs2024thalle,
title={THaLLE: Text Hyperlocally Augmented Large Language Extension -- Technical Report},
author={KBTG Labs and Danupat Khamnuansin and Atthakorn Petchsod and Anuruth Lertpiya and Pornchanan Balee and Thanawat Lodkaew and Tawunrat Chalothorn and Thadpong Pongthawornkamol and Monchai Lertsutthiwong},
year={2024},
eprint={2406.07505},
archivePrefix={arXiv},
primaryClass={cs.CL}
}