Views
No views yet
| Metric | hellaswag_it acc_norm | arc_it acc_norm | m_mmlu_it 5-shot acc | Average |
|---|---|---|---|---|
| Accuracy Normalized | 0.5841 | 0.4414 | 0.5389 | 0.5214 |
1from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
2import torch
3
4device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
5
6tokenizer = AutoTokenizer.from_pretrained("FairMind/Phi-3-mini-4k-instruct-bnb-4bit-Ita")
7model = AutoModelForCausalLM.from_pretrained("FairMind/Phi-3-mini-4k-instruct-bnb-4bit-Ita")
8model.to(device)
9
10
11generation_config = GenerationConfig(
12 penalty_alpha=0.6, # The values balance the model confidence and the degeneration penalty in contrastive search decoding.
13 do_sample = True, # Whether or not to use sampling ; use greedy decoding otherwise.
14 top_k=5, # The number of highest probability vocabulary tokens to keep for top-k-filtering.
15 temperature=0.001, # The value used to modulate the next token probabilities.
16 repetition_penalty=1.7, # The parameter for repetition penalty. 1.0 means no penalty.
17 max_new_tokens = 64, # The maximum numbers of tokens to generate, ignoring the number of tokens in the prompt.
18 eos_token_id=tokenizer.eos_token_id, # The id of the *end-of-sequence* token.
19 pad_token_id=tokenizer.eos_token_id, # The id of the *padding* token.
20 )
21
22
23def generate_answer(question):
24 messages = [
25 {"role": "user", "content": question},
26 ]
27 model_inputs = tokenizer.apply_chat_template(messages, return_tensors="pt").to(device)
28 outputs = model.generate(model_inputs, generation_config=generation_config)
29 result = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
30 return result
31
32
33question = """Quale è la torre più famosa di Parigi?"""
34answer = generate_answer(question)
35print(answer)