#!/bin/bash

# Activate the conda environment
source ~/miniconda3/etc/profile.d/conda.sh
conda activate exllamav2

# Set the model name and bit size
MODEL_NAME="mixtral-8x22b-instruct-oh"
BIT_PRECISIONS=(7.0 6.0 5.5 5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)

# Print the markdown table header
echo "| Quant Level | Perplexity Score |"
echo "|-------------|------------------|"

for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
do
  MODEL_DIR="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"
  if [ -d "$MODEL_DIR" ]; then
    output=$(python test_inference.py -m "$MODEL_DIR" -gs 44,48 -ed data/wikitext/wikitext-2-v1.parquet)
    score=$(echo "$output" | grep -oP 'Evaluation perplexity: \K[\d.]+')
    echo "| $BIT_PRECISION | $score |"
  fi
done

Quant Details

This is the script used for quantization.

bash

#!/bin/bash

# Activate the conda environment
source ~/miniconda3/etc/profile.d/conda.sh
conda activate exllamav2

# Set the model name and bit size
MODEL_NAME="mixtral-8x22b-instruct-oh"

# Define variables
MODEL_DIR="models/$MODEL_NAME"
OUTPUT_DIR="exl2_$MODEL_NAME"
MEASUREMENT_FILE="measurements/$MODEL_NAME.json"

# Create the measurement file if needed
if [ ! -f "$MEASUREMENT_FILE" ]; then
    echo "Creating $MEASUREMENT_FILE"
    # Create directories
    if [ -d "$OUTPUT_DIR" ]; then
        rm -r "$OUTPUT_DIR"
    fi
    mkdir "$OUTPUT_DIR"

    python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -om $MEASUREMENT_FILE
fi

# Choose one of the below. Either create a single quant for testing or a batch of them.
# BIT_PRECISIONS=(2.25)
BIT_PRECISIONS=(7.0 6.0 5.5 5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)

for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
do
    CONVERTED_FOLDER="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"

    # If it doesn't already exist, make the quant
    if [ ! -d "$CONVERTED_FOLDER" ]; then

        echo "Creating $CONVERTED_FOLDER"

        # Create directories
        if [ -d "$OUTPUT_DIR" ]; then
            rm -r "$OUTPUT_DIR"
        fi
        mkdir "$OUTPUT_DIR"
        mkdir "$CONVERTED_FOLDER"
        
        # Run conversion commands  
        python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -m $MEASUREMENT_FILE -b $BIT_PRECISION -cf $CONVERTED_FOLDER

    fi
done

Dracones/mixtral-8x22b-instruct-oh_exl2_2.5bpw

mixtral-8x22b-instruct-oh - EXL2 2.5bpw

EXL2 Version

Perplexity Scoring

Perplexity Script

Quant Details

Dracones/
mixtral-8x22b-instruct-oh_exl2_2.5bpw