Views
No views yet
onnxruntime-genai; model architecture has a non-standard head size of 120 which is not divisible by 16.hf download h2oai/h2o-danube3-4b-chat --local-dir ~/.onnx/h2o-danube3-4b-chat1from optimum.onnxruntime import ORTModelForCausalLM
2import torch
3
4model_id = "~/.onnx/h2o-danube3-4b-chat"
5
6print(f"Loading {model_id} in FP16 to save RAM...")
7
8# 1. Load model in FP16 directly (Crucial for preventing crashes)
9model = ORTModelForCausalLM.from_pretrained(
10 model_id,
11 export=True,
12 torch_dtype=torch.float16, # This tells PyTorch to use half the memory
13 trust_remote_code=True
14)
15
16# 2. Save the ONNX model
17print("Exporting to ONNX...")
18model.save_pretrained("~/.onnx/h2o-danube3-4b-chat-onnx")
19print("Done!")1python -m onnxruntime_genai.models.builder \
2 -m ~/.onnx/h2o-danube3-4b-chat-onnx \
3 -o ~/.onnx/h2o-danube3-4b-chat-onnx-int4-cpu \
4 -p int4 \
5 -e cpu