Views
No views yet
--fast option and Patch Sage Attention KJ node(AUTO).| Quant | it/s | Time (s) | Speed vs BF16 (%) |
|---|---|---|---|
| BF16 | 4.65 | 11.70 | 0.00% |
| Q8_0 | 4.46 | 12.07 | -4.09% |
| Q6_K | 3.60 | 14.91 | -22.58% |
| Q5_K_S | 3.35 | 15.94 | -28.03% |
| Q5_K_M | 3.41 | 15.67 | -26.67% |
| Q5_1 | 3.42 | 15.24 | -26.45% |
| Q5_0 | 3.40 | 15.73 | -26.88% |
| Q4_K_S | 3.55 | 15.12 | -23.66% |
| Q4_K_M | 3.59 | 14.98 | -22.80% |
| Q4_1 | 4.01 | 13.46 | -13.76% |
| Q4_0 | 3.97 | 13.50 | -14.62% |
1import torch
2import safetensors.torch
3import os
4import sys
5
6def convert_to_fp32(input_path, output_path):
7 state_dict = safetensors.torch.load_file(input_path)
8
9 new_state_dict = {}
10 for key, tensor in state_dict.items():
11 print(f"{key} ({tensor.dtype}) -> torch.float32")
12 new_tensor = tensor.to(torch.float32)
13 new_state_dict[key] = new_tensor
14
15 safetensors.torch.save_file(new_state_dict, output_path)
16 print(f"output_path: {output_path}")
17
18if __name__ == "__main__":
19 assert len(sys.argv) == 3, f"usage: {sys.argv[0]} SOURCE TARGET"
20 input_path, output_path = sys.argv[1:3]
21
22 convert_to_fp32(input_path, output_path)llama-quantize.