GGUF quantized versions of
YuanLabAI/Yuan3.0-Flash , a 40B parameter multimodal MoE model (~3.7B activated).
1 # Pull the latest image
2 docker pull ghcr.io/qades/llama.cpp:latest
3
4 # Run with GPU
5 docker run --gpus all -v /home/mk/yuan/Yuan3.0-Flash-GGUF:/model \
6 ghcr.io/qades/llama.cpp:latest \
7 ./llama-cli -m /model/Yuan3.0-Flash-Q4_K_M.gguf \
8 --mmproj /model/mmproj-Yuan3.0-Flash-f16.gguf \
9 -c 131072 -n 4096 --temp 0.7
10
11 # Or use the OAI-compatible server
12 docker run --gpus all -p 8080 :8080 -v /home/mk/yuan/Yuan3.0-Flash-GGUF:/model \
13 ghcr.io/qades/llama.cpp:latest \
14 ./llama-server -m /model/Yuan3.0-Flash-Q4_K_M.gguf \
15 --mmproj /model/mmproj-Yuan3.0-Flash-f16.gguf -c 131072
1 # Clone the custom branch
2 cd ~/llama.cpp
3 git checkout yuan3_0
4 cmake -B build -DCMAKE_BUILD_TYPE = Release
5 cmake --build build --config Release
6
7 # Run
8 ./build/bin/llama-cli -m .. /Yuan3.0-Flash-GGUF/Yuan3.0-Flash-Q4_K_M.gguf \
9 --mmproj .. /Yuan3.0-Flash-GGUF/mmproj-Yuan3.0-Flash-f16.gguf \
10 -c 131072 -n 4096 --temp 0.7
1 from llama_cpp import Llama
2
3 llm = Llama (
4 model_path = "Yuan3.0-Flash-Q4_K_M.gguf" ,
5 mmproj_path = "mmproj-Yuan3.0-Flash-f16.gguf" ,
6 n_ctx = 131072 ,
7 n_gpu_layers = - 1 ,
8 )
9
10 # Text-only
11 output = llm ( "Explain quantum computing in simple terms" )
12
13 # With image
14 from llama_cpp import LlamaVision
15
16 llm = LlamaVision (
17 model_path = "Yuan3.0-Flash-Q4_K_M.gguf" ,
18 mmproj_path = "mmproj-Yuan3.0-Flash-f16.gguf" ,
19 )
20 output = llm ( [ { "type" : "image" , "image" : "photo.jpg" } , { "type" : "text" , "text" : "What do you see?" } ] )
1 FROM ./Yuan3.0 - Flash - Q4_K_M.gguf
2 PARAMETER mmproj ./mmproj - Yuan3.0 - Flash - f16.gguf
3 PARAMETER context_length 131072
4 PARAMETER temperature 0.7
1 ollama create yuan3.0-flash -f Modelfile
2 ollama run yuan3.0-flash
1 vllm serve YuanLabAI/Yuan3.0-Flash \
2 --dtype half \
3 --max-model-len 131072
1 @software{yuan3.0flash_gguf,
2 title = {Yuan3.0-Flash-GGUF},
3 author = {Michael Klaus},
4 year = {2025},
5 url = {https://huggingface.co/YuanLabAI/Yuan3.0-Flash}
6 }
1 @misc{yuan3.0flash,
2 title = {Yuan3.0 Flash: An Open Multimodal Large Language Model for Enterprise Applications},
3 author = {YuanLab AI},
4 year = {2025},
5 eprint = {2601.01718},
6 archivePrefix = {arXiv},
7 primaryClass = {cs.CL}
8 }
Yuan3.0-Flash-GGUF/
├── mmproj-Yuan3.0-Flash-f16.gguf # Vision projector (~744MB)
├── Yuan3.0-Flash-f16-00001-of-00003.gguf # F16 shard 1 (~21GB)
├── Yuan3.0-Flash-f16-00002-of-00003.gguf # F16 shard 2 (~27GB)
├── Yuan3.0-Flash-f16-00003-of-00003.gguf # F16 shard 3 (~29GB)
└── Yuan3.0-Flash-Q4_K_M.gguf # Q4_K_M quantization (~23GB)
