Gausson/

pythia-160m-deduped-n64-RoBiPE-SepLLM

CUDA_LAUNCH_BLOCKING=1
lm_eval --model hf \
	--model_args pretrained=Gausson/pythia-160m-deduped-n64-RoBiPE-SepLLM \
	--tasks  arc_challenge,arc_easy,lambada_openai,logiqa,piqa,sciq,winogrande,wsc,wikitext  \
	--num_fewshot 5 \
	--device cuda:0\
	--batch_size 32

hf (pretrained=Gausson/pythia-160m-deduped-n64-RoBiPE-SepLLM), gen_kwargs: (), limit: None, num_fewshot: 5, batch_size: 32
|    Tasks     |Version|Filter|n-shot|    Metric     | Value  |   |Stderr|
|--------------|-------|------|-----:|---------------|-------:|---|-----:|
|arc_challenge |Yaml   |none  |     5|acc            |  0.2048|±  |0.0118|
|              |       |none  |     5|acc_norm       |  0.2355|±  |0.0124|
|arc_easy      |Yaml   |none  |     5|acc            |  0.4668|±  |0.0102|
|              |       |none  |     5|acc_norm       |  0.4432|±  |0.0102|
|lambada_openai|Yaml   |none  |     5|perplexity     | 38.0503|±  |1.2942|
|              |       |none  |     5|acc            |  0.3051|±  |0.0064|
|logiqa        |Yaml   |none  |     5|acc            |  0.2396|±  |0.0167|
|              |       |none  |     5|acc_norm       |  0.2642|±  |0.0173|
|piqa          |Yaml   |none  |     5|acc            |  0.6436|±  |0.0112|
|              |       |none  |     5|acc_norm       |  0.6366|±  |0.0112|
|sciq          |Yaml   |none  |     5|acc            |  0.8090|±  |0.0124|
|              |       |none  |     5|acc_norm       |  0.7880|±  |0.0129|
|wikitext      |Yaml   |none  |     5|word_perplexity|168.1908|   |      |
|              |       |none  |     5|byte_perplexity|  2.6076|   |      |
|              |       |none  |     5|bits_per_byte  |  1.3827|   |      |
|winogrande    |Yaml   |none  |     5|acc            |  0.4964|±  |0.0141|
|wsc           |Yaml   |none  |     5|acc            |  0.4519|±  |0.0490|

@inproceedings{chen2025sepllm,
  title={{SepLLM: Accelerate Large Language Models by Compressing One Segment into One Separator}},
  author={Chen, Guoxuan and Shi, Han and Li, Jiawei and Gao, Yihang and Ren, Xiaozhe and Chen, Yimeng and Jiang, Xin and Li, Zhenguo and Liu, Weiyang and Huang, Chao},
  booktitle={International Conference on Machine Learning},
  year={2025},
  note={Also available at arXiv:2412.12094}
}