You can deploy this model and start earning money today!

robinshao/
safe_huihui_qwen3_5_9b_mxfp4_gguf

Name: safe_huihui_qwen3_5_9b_mxfp4_gguf
Availability: InStock

gguf

endpoints_compatible

conversational

Views

No views yet

robinshao/safe_huihui_qwen3_5_9b_mxfp4_gguf

Source model: robinshao/safe_huihui_qwen3_5_9b

This repo contains:

safe_huihui_qwen3_5_9b_mxfp4_gguf.gguf: GGUF produced with the local patched llama.cpp toolchain
llama.cpp.src-patched.zip: current uncompiled patched source tree
llama.cpp.build-local-bin.zip: compiled binaries matching the source tree
llama.cpp.build-windows-cuda-local-bin.zip: additional compiled patched llama.cpp binaries
build_result.json: build, quantization, and smoke-test summary

Quantization summary:

json

{
  "arch": "qwen35",
  "tensor_count": 427,
  "type_counts": {
    "Q8_0": 2,
    "F32": 177,
    "MXFP4": 248
  },
  "converted_preview": [
    {
      "index": 1,
      "name": "output.weight",
      "from": "BF16",
      "to": "Q8_0",
      "bytes": 1080688640
    },
    {
      "index": 2,
      "name": "token_embd.weight",
      "from": "BF16",
      "to": "Q8_0",
      "bytes": 1080688640
    },
    {
      "index": 3,
      "name": "blk.0.attn_norm.weight",
      "from": "F32",
      "to": "F32",
      "bytes": 16384
    },
    {
      "index": 4,
      "name": "blk.0.ssm_a",
      "from": "F32",
      "to": "F32",
      "bytes": 128
    },
    {
      "index": 5,
      "name": "blk.0.ssm_conv1d.weight",
      "from": "F32",
      "to": "F32",
      "bytes": 131072
    },
    {
      "index": 6,
      "name": "blk.0.ssm_dt.bias",
      "from": "F32",
      "to": "F32",
      "bytes": 128
    },
    {
      "index": 7,
      "name": "blk.0.ssm_alpha.weight",
      "from": "BF16",
      "to": "MXFP4",
      "bytes": 69632
    },
    {
      "index": 8,
      "name": "blk.0.ssm_beta.weight",
      "from": "BF16",
      "to": "MXFP4",
      "bytes": 69632
    },
    {
      "index": 9,
      "name": "blk.0.attn_qkv.weight",
      "from": "BF16",
      "to": "MXFP4",
      "bytes": 17825792
    },
    {
      "index": 10,
      "name": "blk.0.attn_gate.weight",
      "from": "BF16",
      "to": "MXFP4",
      "bytes": 8912896
    },
    {
      "index": 11,
      "name": "blk.0.ssm_norm.weight",
      "from": "F32",
      "to": "F32",
      "bytes": 512
    },
    {
      "index": 12,
      "name": "blk.0.ssm_out.weight",
      "from": "BF16",
      "to": "MXFP4",
      "bytes": 8912896
    },
    {
      "index": 13,
      "name": "blk.0.ffn_down.weight",
      "from": "BF16",
      "to": "MXFP4",
      "bytes": 26738688
    },
    {
      "index": 14,
      "name": "blk.0.ffn_gate.weight",
      "from": "BF16",
      "to": "MXFP4",
      "bytes": 26738688
    },
    {
      "index": 15,
      "name": "blk.0.ffn_up.weight",
      "from": "BF16",
      "to": "MXFP4",
      "bytes": 26738688
    },
    {
      "index": 16,
      "name": "blk.0.post_attention_norm.weight",
      "from": "F32",
      "to": "F32",
      "bytes": 16384
    },
    {
      "index": 17,
      "name": "blk.1.attn_norm.weight",
      "from": "F32",
      "to": "F32",
      "bytes": 16384
    },
    {
      "index": 18,
      "name": "blk.1.ssm_a",
      "from": "F32",
      "to": "F32",
      "bytes": 128
    },
    {
      "index": 19,
      "name": "blk.1.ssm_conv1d.weight",
      "from": "F32",
      "to": "F32",
      "bytes": 131072
    },
    {
      "index": 20,
      "name": "blk.1.ssm_dt.bias",
      "from": "F32",
      "to": "F32",
      "bytes": 128
    },
    {
      "index": 21,
      "name": "blk.1.ssm_alpha.weight",
      "from": "BF16",
      "to": "MXFP4",
      "bytes": 69632
    },
    {
      "index": 22,
      "name": "blk.1.ssm_beta.weight",
      "from": "BF16",
      "to": "MXFP4",
      "bytes": 69632
    },
    {
      "index": 23,
      "name": "blk.1.attn_qkv.weight",
      "from": "BF16",
      "to": "MXFP4",
      "bytes": 17825792
    },
    {
      "index": 24,
      "name": "blk.1.attn_gate.weight",
      "from": "BF16",
      "to": "MXFP4",
      "bytes": 8912896
    }
  ],
  "output_bytes": 5852045632
}

Smoke test summary:

json

{
  "ok": true,
  "returncode": 0,
  "timed_out": false,
  "load_seen": false,
  "elapsed_sec": 2.42,
  "stdout_tail": ",\n\n",
  "stderr_tail": "m_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)\n0.02.109.491 I llama_completion: llama threadpool init, n_threads = 32\n0.02.112.200 I \n0.02.112.276 I system_info: n_threads = 32 (n_threads_batch = 32) / 64 | CUDA : ARCHS = 1200 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | BLACKWELL_NATIVE_FP4 = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | \n0.02.112.279 I \n0.02.112.360 I sampler seed: 3613287308\n0.02.112.369 I sampler params: \n\trepeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000\n\tdry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = -1\n\ttop_k = 40, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000\n\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900\n0.02.112.374 I sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist \n0.02.112.375 I generate: n_ctx = 256, n_batch = 2048, n_predict = 1, n_keep = 0\n0.02.112.376 I \n0.02.123.029 I common_perf_print:    sampling time =       1.77 ms\n0.02.123.030 I common_perf_print:    samplers time =       0.09 ms /     2 tokens\n0.02.123.033 I common_perf_print:        load time =    1244.71 ms\n0.02.123.034 I common_perf_print: prompt eval time =       0.00 ms /     1 tokens (    0.00 ms per token,      inf tokens per second)\n0.02.123.035 I common_perf_print:        eval time =       8.86 ms /     1 runs   (    8.86 ms per token,   112.92 tokens per second)\n0.02.123.036 I common_perf_print:       total time =      13.54 ms /     2 tokens\n0.02.123.037 I common_perf_print: unaccounted time =       2.92 ms /  21.6 %      (total - sampling - prompt eval - eval) / (total)\n0.02.123.040 I common_perf_print:    graphs reused =          0\n",
  "ctx_size": 256,
  "ngl": 999,
  "predict_tokens": 1,
  "timeout_sec": 300,
  "gpu_layers_forced": true,
  "stdout_log": "/data/cz_qwen_mxfp4_work/runs/robinshao__safe_huihui_qwen3_5_9b/smoke_logs/robinshao__safe_huihui_qwen3_5_9b-mxfp4.out.log",
  "stderr_log": "/data/cz_qwen_mxfp4_work/runs/robinshao__safe_huihui_qwen3_5_9b/smoke_logs/robinshao__safe_huihui_qwen3_5_9b-mxfp4.err.log"
}