#!/bin/bash

# Activate the conda environment
source ~/miniconda3/etc/profile.d/conda.sh
conda activate exllamav2

DATA_SET=/root/wikitext/wikitext-2-v1.parquet

# Set the model name and bit size
MODEL_NAME="WizardLM-2-8x22B"
BIT_PRECISIONS=(6.0 5.5 5.0 4.5 4.0 3.5 3.25 3.0 2.75 2.5 2.25)

# Print the markdown table header
echo "| Quant Level | Perplexity Score |"
echo "|-------------|------------------|"

for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
do
  LOCAL_FOLDER="/root/models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"
  REMOTE_FOLDER="Dracones/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"

  if [ ! -d "$LOCAL_FOLDER" ]; then
    huggingface-cli download --local-dir-use-symlinks=False --local-dir "${LOCAL_FOLDER}" "${REMOTE_FOLDER}" >> /root/download.log 2>&1
  fi

  output=$(python test_inference.py -m "$LOCAL_FOLDER" -gs 40,40,40,40 -ed "$DATA_SET")
  score=$(echo "$output" | grep -oP 'Evaluation perplexity: \K[\d.]+')
  echo "| $BIT_PRECISION | $score |"
  # rm -rf "${LOCAL_FOLDER}"
done

Quant Details

This is the script used for quantization.

bash

#!/bin/bash

# Activate the conda environment
source ~/miniconda3/etc/profile.d/conda.sh
conda activate exllamav2

# Set the model name and bit size
MODEL_NAME="WizardLM-2-8x22B"

# Define variables
MODEL_DIR="/mnt/storage/models/$MODEL_NAME"
OUTPUT_DIR="exl2_$MODEL_NAME"
MEASUREMENT_FILE="measurements/$MODEL_NAME.json"

# Create the measurement file if needed
if [ ! -f "$MEASUREMENT_FILE" ]; then
    echo "Creating $MEASUREMENT_FILE"
    # Create directories
    if [ -d "$OUTPUT_DIR" ]; then
        rm -r "$OUTPUT_DIR"
    fi
    mkdir "$OUTPUT_DIR"

    python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -om $MEASUREMENT_FILE
fi

# Choose one of the below. Either create a single quant for testing or a batch of them.
# BIT_PRECISIONS=(2.25)
BIT_PRECISIONS=(5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)

for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
do
    CONVERTED_FOLDER="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"

    # If it doesn't already exist, make the quant
    if [ ! -d "$CONVERTED_FOLDER" ]; then

        echo "Creating $CONVERTED_FOLDER"

        # Create directories
        if [ -d "$OUTPUT_DIR" ]; then
            rm -r "$OUTPUT_DIR"
        fi
        mkdir "$OUTPUT_DIR"
        mkdir "$CONVERTED_FOLDER"
        
        # Run conversion commands  
        python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -m $MEASUREMENT_FILE -b $BIT_PRECISION -cf $CONVERTED_FOLDER

    fi
done

Dracones/WizardLM-2-8x22B_exl2_3.75bpw

WizardLM-2-8x22B - EXL2 3.75bpw

EXL2 Version

Perplexity Scoring

Perplexity Script

Quant Details

Dracones/
WizardLM-2-8x22B_exl2_3.75bpw