Views
No views yet
mii-llm/zagreus-0.4B-ita for the Italian Post-Training Challenge.sapienzanlp/mmlu_italian splits fewshot and validations-conia/mmlu_italian split validationli-lab/MMLU-ProX-Lite config it, split validation1{
2 "base_model": "mii-llm/zagreus-0.4B-ita",
3 "output_repo": "FinancialSupport/zagreus-0.4b-ita-italic-sft-codex",
4 "train_mode": "dpo",
5 "target_style": "letter",
6 "baseline_quick": null,
7 "variants": [
8 {
9 "variant": {
10 "name": "dpo_beta010_lr1e-6_ep1",
11 "learning_rate": 1e-06,
12 "epochs": 1.0,
13 "beta": 0.1,
14 "nll_weight": 0.05,
15 "batch_size": 2,
16 "gradient_accumulation_steps": 16
17 },
18 "model_dir": "/tmp/zagreus_italic_work/runs/dpo_beta010_lr1e-6_ep1",
19 "quick_eval": null
20 }
21 ],
22 "best_variant": {
23 "name": "dpo_beta010_lr1e-6_ep1",
24 "learning_rate": 1e-06,
25 "epochs": 1.0,
26 "beta": 0.1,
27 "nll_weight": 0.05,
28 "batch_size": 2,
29 "gradient_accumulation_steps": 16
30 },
31 "full_eval": {
32 "label": "dpo_beta010_lr1e-6_ep1_full",
33 "limit": "null",
34 "result_path": "/tmp/zagreus_italic_work/eval_results/dpo_beta010_lr1e-6_ep1_full/custom_openai_eval_model_italic_fast_few_shot.json",
35 "metrics": {
36 "accuracy": 0.292,
37 "accuracy:std": 0.45468230667137244
38 },
39 "invalid_answers": 0,
40 "prediction_counts": {
41 "B": 4172,
42 "A": 5821,
43 "C": 6,
44 "D": 1
45 },
46 "macro_accuracy": {
47 "culture and commonsense": {
48 "accuracy": 0.2918203151190077,
49 "n": 5966
50 },
51 "language capability": {
52 "accuracy": 0.2922657411998017,
53 "n": 4034
54 }
55 },
56 "category_accuracy": {
57 "art_history": {
58 "accuracy": 0.2836734693877551,
59 "n": 980
60 },
61 "civic_education": {
62 "accuracy": 0.31654676258992803,
63 "n": 973
64 },
65 "current_events": {
66 "accuracy": 0.32608695652173914,
67 "n": 92
68 },
69 "geography": {
70 "accuracy": 0.2522982635342186,
71 "n": 979
72 },
73 "history": {
74 "accuracy": 0.2607361963190184,
75 "n": 978
76 },
77 "lexicon": {
78 "accuracy": 0.27987742594484166,
79 "n": 979
80 },
81 "literature": {
82 "accuracy": 0.2845528455284553,
83 "n": 984
84 },
85 "morphology": {
86 "accuracy": 0.2785714285714286,
87 "n": 140
88 },
89 "orthography": {
90 "accuracy": 0.282183316168898,
91 "n": 971
92 },
93 "synonyms_and_antonyms": {
94 "accuracy": 0.32852729145211124,
95 "n": 971
96 },
97 "syntax": {
98 "accuracy": 0.2805755395683453,
99 "n": 973
100 },
101 "tourism": {
102 "accuracy": 0.35,
103 "n": 980
104 }
105 }
106 },
107 "train_records": 13074,
108 "tokenized_train_records": 29976,
109 "italic_revision": "92df420ff686babeea54e217b9f90f8471374916",
110 "max_length": 1536,
111 "use_eval_few_shots": true,
112 "augment_rotations": 5,
113 "four_option_only": false,
114 "include_extra_mcq": false,
115 "global_mmlu_limit": 6000
116}