Views
No views yet
1# !pip install llama-cpp-python
2from llama_cpp import Llama
3
4llm = Llama.from_pretrained(
5 repo_id="sodeeplearning/pozdgpt",
6 filename="PozdGPT-Q4_K_M.gguf", # Or Q6_K, Q8_0, f16
7)1# !pip install vllm
2
3vllm serve ./PozdGPT-awq-4bit \
4 --served-model-name pozdgpt \
5 --quantization compressed-tensors \
6 --max-model-len 8192 \
7 --gpu-memory-utilization 0.88 \
8 --max-num-seqs 6 \
9 --kv-cache-dtype fp8 \
10 --enable-prefix-caching \
11 --api-key key \
12 --port 8148