This is an INT8 quantized version of Microsoft's BioGPT for CPU inference.
1import torch
2from transformers import AutoTokenizer, AutoModelForCausalLM
3
4# Load base model and apply quantization
5tokenizer = AutoTokenizer.from_pretrained("microsoft/biogpt")
6model = AutoModelForCausalLM.from_pretrained("microsoft/biogpt", torch_dtype=torch.float16)
7model = torch.quantization.quantize_dynamic(model, {torch.nn.Linear}, dtype=torch.qint8)
8model.eval()
9
10# Use for inference
11prompt = "Extract medical features: Patient is 45-year-old male with fever 101.2F"
12inputs = tokenizer.encode(prompt, return_tensors="pt")
13outputs = model.generate(inputs, max_new_tokens=100)
14print(tokenizer.decode(outputs[0], skip_special_tokens=True))
