Views
No views yet
pip install auto-gptq[triton]==0.4.2 transformers1from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
2from transformers import AutoTokenizer
3
4model_name_or_path = "mmnga/cyberagent-calm2-7b-GPTQ-calib-ja-1k"
5
6# Tokenizer
7tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True)
8
9# Model
10model = AutoGPTQForCausalLM.from_quantized(model_name_or_path, use_safetensors=True, device="cuda:0")
11
12# Your test prompt
13prompt = """今日の夕食のレシピをご紹介します。"""
14print(tokenizer.decode(model.generate(**tokenizer(prompt, return_tensors="pt").to(model.device), max_length=128)[0]))