Views
No views yet
1# Base ctransformers with CUDA GPU acceleration
2! pip install ctransformers[cuda]>=0.2.24
3# Or with no GPU acceleration
4# ! pip install ctransformers>=0.2.24
5! pip install -U sentence-transformers
6! pip install transformers huggingface_hub torch
71from ctransformers import AutoModelForCausalLM
2from transformers import pipeline, AutoModel, AutoTokenizer
3from sentence_transformers import SentenceTransformer
4import os1
2# Load LLM and Tokenizer
3
4model_mistral = AutoModelForCausalLM.from_pretrained(
5 "alokabhishek/Mistral-7B-Instruct-v0.2-GGUF",
6 model_file="mistral-7b-instruct-v0.2.Q4_K_M.gguf", # replace Q4_K_M.gguf with Q5_K_M.gguf as needed
7 model_type="mistral",
8 gpu_layers=50, # Use `gpu_layers` to specify how many layers will be offloaded to the GPU.
9 hf=True
10)
11
12
13tokenizer_mistral = AutoTokenizer.from_pretrained(
14 "alokabhishek/Mistral-7B-Instruct-v0.2-GGUF", use_fast=True
15)
16
17# Create a pipeline
18pipe_mistral = pipeline(model=model_mistral, tokenizer=tokenizer_mistral, task='text-generation')
19
20prompt_mistral = "Tell me a funny joke about Large Language Models meeting a Blackhole in an intergalactic Bar."
21
22
23output_mistral = pipe_mistral(prompt_mistral, max_new_tokens=512)
24print(output_mistral[0]["generated_text"])
25