Views
No views yet
| Name | Quant method | Size |
|---|---|---|
| karasu-7B.Q2_K.gguf | Q2_K | 2.92GB |
| karasu-7B.IQ3_XS.gguf | IQ3_XS | 3.23GB |
| karasu-7B.IQ3_S.gguf | IQ3_S | 3.39GB |
| karasu-7B.Q3_K_S.gguf | Q3_K_S | 3.37GB |
| karasu-7B.IQ3_M.gguf | IQ3_M | 3.48GB |
| karasu-7B.Q3_K.gguf | Q3_K | 3.7GB |
| karasu-7B.Q3_K_M.gguf | Q3_K_M | 3.7GB |
| karasu-7B.Q3_K_L.gguf | Q3_K_L | 3.98GB |
| karasu-7B.IQ4_XS.gguf | IQ4_XS | 4.13GB |
| karasu-7B.Q4_0.gguf | Q4_0 | 4.29GB |
| karasu-7B.IQ4_NL.gguf | IQ4_NL | 4.34GB |
| karasu-7B.Q4_K_S.gguf | Q4_K_S | 4.32GB |
| karasu-7B.Q4_K.gguf | Q4_K | 4.54GB |
| karasu-7B.Q4_K_M.gguf | Q4_K_M | 4.54GB |
| karasu-7B.Q4_1.gguf | Q4_1 | 4.73GB |
| karasu-7B.Q5_0.gguf | Q5_0 | 5.16GB |
| karasu-7B.Q5_K_S.gguf | Q5_K_S | 5.16GB |
| karasu-7B.Q5_K.gguf | Q5_K | 5.29GB |
| karasu-7B.Q5_K_M.gguf | Q5_K_M | 5.29GB |
| karasu-7B.Q5_1.gguf | Q5_1 | 5.6GB |
| karasu-7B.Q6_K.gguf | Q6_K | 6.09GB |
| karasu-7B.Q8_0.gguf | Q8_0 | 7.88GB |
1from transformers import AutoTokenizer, AutoModelForCausalLM
2import torch
3
4tokenizer = AutoTokenizer.from_pretrained("lightblue/karasu-7B")
5model = AutoModelForCausalLM.from_pretrained("lightblue/karasu-7B", torch_dtype=torch.bfloat16, device_map="auto")
6
7pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
8
9messages = [{"role": "system", "content": "あなたはAIアシスタントです。"}]
10messages.append({"role": "user", "content": "イギリスの首相は誰ですか？"})
11
12prompt = tokenizer.apply_chat_template(conversation=messages, add_generation_prompt=True, tokenize=False)
13
14pipe(prompt, max_new_tokens=100, do_sample=False, temperature=0.0, return_full_text=False)1from vllm import LLM, SamplingParams
2
3sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
4llm = LLM(model="lightblue/karasu-7B")
5
6messages = [{"role": "system", "content": "あなたはAIアシスタントです。"}]
7messages.append({"role": "user", "content": "イギリスの首相は誰ですか？"})
8prompt = llm.llm_engine.tokenizer.apply_chat_template(conversation=messages, add_generation_prompt=True, tokenize=False)
9prompts = [prompt]
10
11outputs = llm.generate(prompts, sampling_params)
12for output in outputs:
13 prompt = output.prompt
14 generated_text = output.outputs[0].text
15 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")