Views
No views yet
1given the text
2text: {text}
3
4normalize to {language} languagetext is the text you want to normalize.language is language you want to normalize, you can omit normalize to {language} language this make the model normalize based on the text language.{"normalized_text": "All suspects, aged twenty five to thirty seven, were remanded for seven days beginning today to facilitate investigations under Sections twelve open parenthesis two close parenthesis, thirty nine A open parenthesis one close parenthesis and thirty nine A open parenthesis two close parenthesis of the Dangerous Drugs Act one thousand nine hundred fifty two. dash Bernama", "normalizer_mapping": {"25": "twenty five", "37": "thirty seven", "12(2)": "twelve open parenthesis two close parenthesis", "39A(1)": "thirty nine A open parenthesis one close parenthesis", "39A(2)": "thirty nine A open parenthesis two close parenthesis", "1952": "one thousand nine hundred fifty two", "\u2014": "dash"}}1from transformers import TextStreamer, AutoModelForCausalLM, AutoTokenizer
2import transformers
3import torch
4
5model = AutoModelForCausalLM.from_pretrained(
6 'malaysia-ai/Malaysian-Normalizer-Qwen3-8B',
7 torch_dtype='auto'
8).cuda()
9tokenizer = AutoTokenizer.from_pretrained('malaysia-ai/Malaysian-Normalizer-Qwen3-8B')
10
11user = """
12given the text
13text: “Oleochemical exports dropped 2.72 per cent m-o-m to 210,924 tonnes from 216,816 tonnes while biodiesel exports fell 48.89 per cent m-o-m to 23,689 tonnes from 46,345 tonnes,” it said.
14
15normalize to english language
16"""
17message = [
18 {'role': 'user', 'content': user.strip()}
19]
20prompt = tokenizer.apply_chat_template(message, add_generation_prompt = True, tokenize = False)
21generate_kwargs = dict(
22 **tokenizer(prompt, return_tensors = 'pt').to('cuda'),
23 max_new_tokens=1024,
24 top_p=0.9,
25 top_k=50,
26 temperature=0.9,
27 do_sample=True,
28 repetition_penalty=1.0,
29)
30generation_output = model.generate(**generate_kwargs)<|im_start|>user
given the text
text: “Oleochemical exports dropped 2.72 per cent m-o-m to 210,924 tonnes from 216,816 tonnes while biodiesel exports fell 48.89 per cent m-o-m to 23,689 tonnes from 46,345 tonnes,” it said.
normalize to english language<|im_end|>
<|im_start|>assistant
<think>
</think>
{"normalized_text": "open quote Oleochemical exports dropped two point seven two per cent m dash o dash m to two hundred ten thousand nine hundred twenty four tonnes from two hundred sixteen thousand eight hundred sixteen tonnes while biodiesel exports fell forty eight point eight nine per cent m dash o dash m to twenty three thousand six hundred eighty nine tonnes from forty six thousand three hundred forty five tonnes, close quote it said.", "normalizer_mapping": {"\u201c": "open quote", "2.72": "two point seven two", "m-o-m": "m dash o dash m", "210,924": "two hundred ten thousand nine hundred twenty four", "216,816": "two hundred sixteen thousand eight hundred sixteen", "48.89": "forty eight point eight nine", "23,689": "twenty three thousand six hundred eighty nine", "46,345": "forty six thousand three hundred forty five", "\u201d": "close quote"}}<|im_end|>