import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

# Load base model structure
model = AutoModelForCausalLM.from_pretrained(
    "tokenlabsdotrun/Llama-3.1-8B-ModelOpt-FP8",
    torch_dtype=torch.bfloat16,
    low_cpu_mem_usage=True
)

# Load tokenizer and generate
tokenizer = AutoTokenizer.from_pretrained("tokenlabsdotrun/Llama-3.1-8B-ModelOpt-NVFP4-QAT")

inputs = tokenizer("Hello, my name is", return_tensors="pt")
outputs = model.generate(**inputs, max_new_tokens=10)
print(tokenizer.decode(outputs[0], skip_special_tokens=True))

License

This model inherits the Llama 3.1 Community License.

tokenlabsdotrun/Llama-3.1-8B-ModelOpt-FP8-QAT

Llama-3.1-8B-Instruct Quantized (ModelOpt FP8) through QAT

Model Details

Usage

License

tokenlabsdotrun/
Llama-3.1-8B-ModelOpt-FP8-QAT