A quantized model based on TSjB/QM-4B.
QM-4B-AWQ is a language model with an extended tokenizer and fine-tuning for Qarachay-Malqar language support (къарачай-малкъар тил).
1from transformers import AutoModelForCausalLM, AutoTokenizer
2import torch
3
4model = AutoModelForCausalLM.from_pretrained(
5 "TSjB/QM-4B-AWQ",
6 dtype=torch.bfloat16,
7 device_map="auto",
8 trust_remote_code=True
9)
10
11tokenizer = AutoTokenizer.from_pretrained(
12 "TSjB/QM-4B-AWQ",
13 trust_remote_code=True
14)
15
16# With chat template
17messages = [
18{"role": "system", "content": "Сен къарачай-малкъар тилде болушлукъчуса. Соруўлагъа къысха, тюз эм ачыкъ джуўабла бер. Орусча неда ингилизче сорсала — ол тилде джуўаб бер."},
19{"role": "user", "content": "Не зат билесе Къарачай юсюнден?"}
20]
21text = tokenizer.apply_chat_template(
22 messages,
23 tokenize=False,
24 add_generation_prompt=True,
25 enable_thinking=False
26)
27
28inputs = tokenizer(text, return_tensors="pt").to(model.device)
29
30if 'token_type_ids' in inputs:
31 inputs.pop('token_type_ids')
32
33outputs = model.generate(
34 **inputs,
35 max_new_tokens=100,
36 temperature=0.7,
37 top_p=0.9,
38 do_sample=True,
39 repetition_penalty=1.2,
40 no_repeat_ngram_size=4,
41 pad_token_id=tokenizer.pad_token_id,
42 eos_token_id=tokenizer.eos_token_id,
43)
44
45print(tokenizer.decode(outputs[0], skip_special_tokens=True))
1generation_config = {
2 "max_new_tokens": 200,
3 "temperature": 0.7,
4 "top_p": 0.9,
5 "do_sample": True,
6 "repetition_penalty": 1.2, # important to avoid repetitions
7 "no_repeat_ngram_size": 3, # optional
8}
1@misc{qm4bawq202+,
2 title={QM-4B-AWQ: Qarachay-Malqar language support},
3 author={TSjB},
4 year={2026},
5 publisher={HuggingFace},
6 url={https://huggingface.co/TSjB/QM-4B}
7}
