Views
No views yet
1slices:
2 - sources:
3 - model: Hjgugugjhuhjggg/llama-3.2-1B-spinquant-hf
4 layer_range: [1, 1]
5 parameters:
6 weight: 0.3
7 density: 0.2
8 gamma: 0.005
9 normalize: true
10 int8_mask: true
11 random_seed: 42
12 temperature: 0.5
13 top_p: 0.65
14 inference: true
15 max_tokens: 300
16 stream: true
17 quantization:
18 - method: int8
19 value: 60
20 - method: int4
21 value: 40
22merge_method: passthrough
23base_model: huihui-ai/Llama-3.2-1B-Instruct-abliterated
24dtype: float16
25compression:
26 pruning:
27 enabled: true
28 sparsity: 0.95
29 distillation:
30 enabled: true
31 temperature: 0.7
32 model_type: "distilled"
33 quantization:
34 enabled: true
35 methods:
36 - int8
37 - int4
38inference_optimizations:
39 caching:
40 enabled: true
41 cache_size: 1000
42 batching:
43 enabled: true
44 batch_size: 8
45 parallelism:
46 enabled: true
47 workers: 4
48 asynchronous:
49 enabled: true
50 max_concurrent_tasks: 5
51 tensor_cores:
52 enabled: true
53 gpu:
54 enabled: true
55 device: cuda
56 model_sharding:
57 enabled: true
58 shards: 2
59 memory_optimization:
60 enabled: true
61 strategy: "offload"
62 tensor_compression:
63 enabled: true
64 method: "tensor_factorization"
65mixture_of_experts:
66 enabled: true
67 num_experts: 4
68 gating_strategy: top_k
69 top_k: 2
70 load_balancing:
71 enabled: true
72 balance_factor: 0.5
73 expert_capacity:
74 max_tokens_per_expert: 512
75 dynamic_routing:
76 enabled: true
77 routing_threshold: 0.1
78 routing_optimizations:
79 enabled: true
80 cache_routing: true
81model_sparsity:
82 enabled: true
83 sparsity_pattern: "block"
84 mask_method: "random"
85 pruning_factor: 0.98
86auto_tuning:
87 enabled: true
88 batch_size_adaptation:
89 enabled: true
90 factor: 0.8
91 max_batch_size: 32
92 temperature_scheduling:
93 enabled: true
94 start_temp: 1.0
95 end_temp: 0.5
96 schedule: "linear"
97