Quant Size	ChatML	Alpaca	Mistral	Vicuna-v1.1	Vicuna-v0
5.0	79.91	81.45	81.11	78.37	76.64
4.5	80.64	80.9	81.65	77.04	74.6
4.0	80.78	79.53	82.78	79.17	76.41
3.5	81.11	82.42	82.34	81.04	78.09
3.0	79.13	77.74	80.11	79.38	77.25
2.75	79.6	77.85	79.71	76.93	75.91
2.5	77.45	77.0	78.4	75.86	75.25
2.25	77.18	74.06	76.75	75.56	74.28

Perplexity Script

This was the script used for perplexity testing.

bash

#!/bin/bash

# Activate the conda environment
source ~/miniconda3/etc/profile.d/conda.sh
conda activate exllamav2

# Set the model name and bit size
MODEL_NAME="miqu-1-70b-sf"
BIT_PRECISIONS=(8.0 7.0 6.0 5.5 5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)

# Print the markdown table header
echo "| Quant Level | Perplexity Score |"
echo "|-------------|------------------|"

for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
do
  MODEL_DIR="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"
  if [ -d "$MODEL_DIR" ]; then
    output=$(python test_inference.py -m "$MODEL_DIR" -gs 22,24 -ed data/wikitext/wikitext-2-v1.parquet)
    score=$(echo "$output" | grep -oP 'Evaluation perplexity: \K[\d.]+')
    echo "| $BIT_PRECISION | $score |"
  fi
done

Quant Details

This is the script used for quantization.

bash

#!/bin/bash

# Activate the conda environment
source ~/miniconda3/etc/profile.d/conda.sh
conda activate exllamav2

# Set the model name and bit size
MODEL_NAME="miqu-1-70b-sf"

# Define variables
MODEL_DIR="models/152334H_miqu-1-70b-sf"
OUTPUT_DIR="exl2_$MODEL_NAME"
MEASUREMENT_FILE="measurements/$MODEL_NAME.json"

# Create the measurement file if needed
if [ ! -f "$MEASUREMENT_FILE" ]; then
    echo "Creating $MEASUREMENT_FILE"
    # Create directories
    if [ -d "$OUTPUT_DIR" ]; then
        rm -r "$OUTPUT_DIR"
    fi
    mkdir "$OUTPUT_DIR"

    python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -om $MEASUREMENT_FILE
fi

# Choose one of the below. Either create a single quant for testing or a batch of them.
# BIT_PRECISIONS=(5.0)
BIT_PRECISIONS=(8.0 7.0 6.0 5.5 5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)

for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
do
    CONVERTED_FOLDER="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"

    # If it doesn't already exist, make the quant
    if [ ! -d "$CONVERTED_FOLDER" ]; then

        echo "Creating $CONVERTED_FOLDER"

        # Create directories
        if [ -d "$OUTPUT_DIR" ]; then
            rm -r "$OUTPUT_DIR"
        fi
        mkdir "$OUTPUT_DIR"
        mkdir "$CONVERTED_FOLDER"
        
        # Run conversion commands  
        python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -m $MEASUREMENT_FILE -b $BIT_PRECISION -cf $CONVERTED_FOLDER

    fi
done

Dracones/miqu-1-70b-sf_exl2_8.0bpw

miqu-1-70b-sf - EXL2 8.0bpw

EXL2 Version

Perplexity Scoring

EQ Bench

Perplexity Script

Quant Details

Dracones/
miqu-1-70b-sf_exl2_8.0bpw