Views
No views yet
1import os
2if "COLAB_" not in "".join(os.environ.keys()):
3 !pip install unsloth
4else:
5 # Do this only in Colab notebooks! Otherwise use pip install unsloth
6 !pip install --no-deps bitsandbytes accelerate xformers==0.0.29.post3 peft trl==0.15.2 triton cut_cross_entropy unsloth_zoo
7 !pip install sentencepiece protobuf datasets huggingface_hub hf_transfer
8 !pip install --no-deps unsloth1from unsloth import FastLanguageModel
2import torch
3max_seq_length = 5000 # Choose any! We auto support RoPE Scaling internally!
4dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
5load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
6
7model_name = "weifar/unsloth/DeepSeek-R1-Distill-Llama-8B-unsloth-bnb-4bit"
8
9model, tokenizer = FastLanguageModel.from_pretrained(
10 model_name = model_name,
11 max_seq_length = max_seq_length,
12 dtype = dtype,
13 load_in_4bit = load_in_4bit,
14 # token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
15)
16
17model = FastLanguageModel.get_peft_model(
18 model,
19 r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
20 target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
21 "gate_proj", "up_proj", "down_proj",],
22 lora_alpha = 16,
23 lora_dropout = 0, # Supports any, but = 0 is optimized
24 bias = "none", # Supports any, but = "none" is optimized
25 # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
26 use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
27 random_state = 3407,
28 use_rslora = False, # We support rank stabilized LoRA
29 loftq_config = None, # And LoftQ
30)1FastLanguageModel.for_inference(model) # Enable native 2x faster inference
2
3inputs = tokenizer(eval_prompt, return_tensors = "pt").to("cuda")
4
5from transformers import TextStreamer
6text_streamer = TextStreamer(tokenizer)
7_ = model.generate(**inputs, streamer = text_streamer, max_new_tokens = 4000)