git clone https://github.com/mit-han-lab/llm-awq && \
cd llm-awq && \
pip3 install -e . && \
cd awq/kernels && \
python3 setup.py install && \
cd ../.. && \
pip3 install einops

Run:

hfuser="casperhansen"
model_name="mpt-7b-8k-chat-awq"
group_size=128
repo_path="$hfuser/$model_name"
model_path="/workspace/llm-awq/$model_name"
quantized_model_path="/workspace/llm-awq/$model_name/$model_name-w4-g$group_size.pt"

git clone https://huggingface.co/$repo_path

python3 tinychat/demo.py --model_type mpt \
    --model_path $model_path \
    --q_group_size $group_size \
    --load_quant $quantized_model_path \
    --precision W4A16

Citation

Please cite this model using the following format:

@online{MosaicML2023Introducing,
    author    = {MosaicML NLP Team},
    title     = {Introducing MPT-30B: Raising the bar
for open-source foundation models},
    year      = {2023},
    url       = {www.mosaicml.com/blog/mpt-30b},
    note      = {Accessed: 2023-06-22},
    urldate   = {2023-06-22}
}

casperhansen/mpt-7b-8k-chat-awq

MPT-7b-8k-chat

⚡ 4-bit Inference Speed

How to run

Citation

casperhansen/
mpt-7b-8k-chat-awq