Views
No views yet
| Model | Ratio | Stage B (SVD) | Stage F (Fine-tuned) |
|---|---|---|---|
| LLaMA-7B | 20%-80% | llama7b/r{02,04,06,08}/B | llama7b/r{02,04,06,08}/F |
| Qwen3-14B | 20% | qwen3_14b/r02/B | qwen3_14b/r02/F |
from_pretrained(), same size as original model:1from transformers import AutoModelForCausalLM, AutoTokenizer
2
3repo = "zhc12/HLC-compressed-models"
4
5# Qwen3-14B at 80% compression (fine-tuned)
6model = AutoModelForCausalLM.from_pretrained(repo, subfolder="qwen3_14b/r02/F")
7tokenizer = AutoTokenizer.from_pretrained(repo, subfolder="qwen3_14b/r02/F")F/ directory also contains factors.pt with the low-rank factors
A (d*r) and B (r*n) for every compressed linear layer. This is ~20%
smaller than the merged weights and preserves the compression structure.1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer
3from huggingface_hub import hf_hub_download
4
5repo = "zhc12/HLC-compressed-models"
6subfolder = "qwen3_14b/r02/F"
7
8# Load the base merged model
9model = AutoModelForCausalLM.from_pretrained(repo, subfolder=subfolder)
10tokenizer = AutoTokenizer.from_pretrained(repo, subfolder=subfolder)
11
12# Download and load factors
13factors_path = hf_hub_download(repo, f"{subfolder}/factors.pt")
14factors = torch.load(factors_path, map_location="cpu", weights_only=True)
15
16# factors is a dict: {(layer_idx, sublayer_name): {"A": tensor, "B": tensor}}
17# Example: factors[(0, "self_attn.q_proj")]["A"].shape = (5120, 2048)
18print(f"Loaded {len(factors)} factor pairs")1import torch
2import torch.nn as nn
3
4class CompressedLinear(nn.Module):
5 """Low-rank linear: y = A @ (B @ x) + bias, where A is (d, r) and B is (r, n)."""
6 def __init__(self, A, B, bias=None):
7 super().__init__()
8 d, r = A.shape
9 _, n = B.shape
10 self.first = nn.Linear(n, r, bias=False)
11 self.second = nn.Linear(r, d, bias=bias is not None)
12 self.first.weight = nn.Parameter(B)
13 self.second.weight = nn.Parameter(A)
14 if bias is not None:
15 self.second.bias = nn.Parameter(bias)
16
17 def forward(self, x):
18 return self.second(self.first(x.to(self.first.weight.dtype))).to(x.dtype)
19
20# Replace merged layers with factored versions
21for (layer_idx, sublayer_name), f in factors.items():
22 layer = model.model.layers[layer_idx]
23 parts = sublayer_name.split(".")
24 parent = layer
25 for p in parts[:-1]:
26 parent = getattr(parent, p)
27 original = getattr(parent, parts[-1])
28 bias = original.bias.data if original.bias is not None else None
29 compressed = CompressedLinear(f["A"], f["B"], bias=bias)
30 setattr(parent, parts[-1], compressed)
31
32# Now each compressed sublayer has .first.weight (B) and .second.weight (A)
33# Total trainable params = sum of A and B sizes, ~20% fewer than originalq_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_projr = (1 - ratio) * d * n / (d + n)