Views
No views yet
| Metric | Value |
|---|---|
| Accuracy | 0.75 |
| MAE | 0.25 |
| QWK | 0.6575 |
| BERTScore | 0.8871 |
1from transformers import AutoModelForCausalLM, AutoTokenizer
2from peft import PeftModel
3
4base_model = AutoModelForCausalLM.from_pretrained(
5 "unsloth/mistral-7b-instruct-v0.2-bnb-4bit"
6)
7
8model = PeftModel.from_pretrained(
9 base_model,
10 "abooodjbr/autoeval-b837d425-12d7-440b-b997-0ceabda5c37a"
11)
12
13tokenizer = AutoTokenizer.from_pretrained("unsloth/mistral-7b-instruct-v0.2-bnb-4bit")
14
15# Replace the placeholders below with your actual input
16user_content = (
17 "Provide both a score and a rationale by evaluating the student's answer strictly within the mark scheme range, "
18 "grading based on how well it meets the question's requirements by comparing the student answer to the reference answer.\n"
19 "Question: <YOUR_QUESTION>\n"
20 "Reference Answer: <YOUR_REFERENCE_ANSWER>\n"
21 "Student Answer: <YOUR_STUDENT_ANSWER>\n"
22 "Mark Scheme: <YOUR_MARK_SCHEME>" # e.g. {'1': 'Mentions X', '2': 'Explains Y'}
23)
24
25messages = [
26 {
27 "role": "system",
28 "content": "You are a grading assistant. Evaluate student answers based on the mark scheme. Respond only in JSON format with keys 'score' (int) and 'rationale' (string)."
29 },
30 {
31 "role": "user",
32 "content": user_content
33 },
34]
35
36inputs = tokenizer.apply_chat_template(
37 messages,
38 tokenize=True,
39 add_generation_prompt=True,
40 return_tensors="pt",
41 return_dict=True,
42).to(model.device)
43
44generated_ids = model.generate(
45 **inputs,
46 max_new_tokens=128,
47 temperature=0.2,
48 top_k=5,
49 do_sample=False,
50)[0]
51
52new_generated_ids = generated_ids[inputs["input_ids"].shape[1]:]
53generated_text = tokenizer.decode(new_generated_ids, skip_special_tokens=True)
54print(generated_text)
55# Output example: {"score": 4, "rationale": "The student correctly identified..."}