Views
No views yet
1Config(
2 global_quant_config=QuantizationConfig(
3 input_tensors=None,
4 output_tensors=None,
5 weight=QuantizationSpec(
6 dtype=Dtype.uint4,
7 observer_cls=<class 'quark.torch.quantization.observer.observer.PerGroupMinMaxObserver'>,
8 is_dynamic=False,
9 qscheme=QSchemeType.per_group,
10 ch_axis=-1,
11 group_size=128,
12 symmetric=False,
13 round_method=RoundType.half_even,
14 scale_type=ScaleType.float,
15 scale_format=None,
16 scale_calculation_mode=None,
17 qat_spec=None,
18 mx_element_dtype=None,
19 zero_point_type=ZeroPointType.int32,
20 is_scale_quant=False,
21 ),
22 bias=None,
23 target_device=None,
24 ),
25 layer_type_quant_config={},
26 layer_quant_config={},
27 kv_cache_quant_config={},
28 kv_cache_group=['*k_proj', '*v_proj'],
29 min_kv_scale=0.0,
30 softmax_quant_spec=None,
31 exclude=['[]'],
32 algo_config=[
33 AWQConfig(
34 name="awq",
35 scaling_layers=[{'prev_op': 'input_layernorm', 'layers': ['self_attn.q_proj', 'self_attn.k_proj', 'self_attn.v_proj'], 'inp': 'self_attn.q_proj', 'module2inspect': 'self_attn'}, {'prev_op': 'self_attn.v_proj', 'layers': ['self_attn.o_proj'], 'inp': 'self_attn.o_proj'}, {'prev_op': 'post_attention_layernorm', 'layers': ['mlp.gate_proj', 'mlp.up_proj'], 'inp': 'mlp.gate_proj', 'module2inspect': 'mlp'}, {'prev_op': 'mlp.up_proj', 'layers': ['mlp.down_proj'], 'inp': 'mlp.down_proj'}],
36 model_decoder_layers="model.layers",
37 ),
38 ],
39 quant_mode=QuantizationMode.eager_mode,
40 log_severity_level=1,
41 version="0.10",
42)