pip install -q -U bitsandbytes accelerate torch huggingface_hub
pip install -q -U git+https://github.com/huggingface/transformers.git # Install latest version of transformers
pip install -q -U git+https://github.com/huggingface/peft.git
pip install flash-attn --no-build-isolation

Import

python

import torch
import os
from torch import bfloat16
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline, BitsAndBytesConfig, LlamaForCausalLM

Use a pipeline as a high-level helper

python

model_id_mistral = "alokabhishek/Mistral-7B-Instruct-v0.2-bnb-4bit"

tokenizer_mistral = AutoTokenizer.from_pretrained(model_id_mistral, use_fast=True)

model_mistral = AutoModelForCausalLM.from_pretrained(
    model_id_mistral,
    device_map="auto"
)

pipe_mistral = pipeline(model=model_mistral, tokenizer=tokenizer_mistral, task='text-generation')

prompt_mistral = "Tell me a funny joke about Large Language Models meeting a Blackhole in an intergalactic Bar."

output_mistral = pipe_llama(prompt_mistral, max_new_tokens=512)

print(output_mistral[0]["generated_text"])

Uses

Direct Use

[More Information Needed]

Downstream Use [optional]

[More Information Needed]

Out-of-Scope Use

[More Information Needed]

Bias, Risks, and Limitations

[More Information Needed]

Evaluation

Metrics

[More Information Needed]

Results

[More Information Needed]

Model Card Authors [optional]

[More Information Needed]

Model Card Contact

[More Information Needed]