Views
No views yet
pip install pruna1from pruna import PrunaModel
2
3loaded_model = PrunaModel.from_pretrained(
4 "LenSch/torchao"
5)
6# we can then run inference using the methods supported by the base modelsmash_config.json file, which describes the optimization methods that were applied to the model.1{
2 "batcher": null,
3 "cacher": null,
4 "compiler": "torch_compile",
5 "factorizer": null,
6 "kernel": "flash_attn3",
7 "pruner": null,
8 "quantizer": "torchao",
9 "torch_compile_backend": "inductor",
10 "torch_compile_dynamic": null,
11 "torch_compile_fullgraph": false,
12 "torch_compile_make_portable": false,
13 "torch_compile_max_kv_cache_size": 400,
14 "torch_compile_mode": "max-autotune-no-cudagraphs",
15 "torch_compile_seqlen_manual_cuda_graph": 100,
16 "torch_compile_target": "model",
17 "torchao_excluded_modules": "none",
18 "torchao_quant_type": "int8dq",
19 "batch_size": 1,
20 "device": "cuda:0",
21 "device_map": null,
22 "save_fns": [
23 "save_before_apply",
24 "save_before_apply"
25 ],
26 "load_fns": [
27 "diffusers"
28 ],
29 "reapply_after_load": {
30 "factorizer": null,
31 "pruner": null,
32 "quantizer": "torchao",
33 "kernel": "flash_attn3",
34 "cacher": null,
35 "compiler": "torch_compile",
36 "batcher": null
37 }
38}