This is a merge of pre-trained language models created using
mergekit.
This model was merged using the
SLERP merge method.
1# slices:
2# - sources:
3# - model: Cognitive-Lab/LLama3-Gaja-Hindi-8B-v0.1
4# layer_range: [0, 23]
5# - sources:
6# - model: Cognitive-Lab/LLama3-Gaja-Hindi-8B-v0.1
7# layer_range: [24, 28]
8# - model: WizardLM/WizardMath-7B-V1.0
9# layer_range: [24, 28]
10# - sources:
11# - model: Cognitive-Lab/LLama3-Gaja-Hindi-8B-v0.1
12# layer_range: [29, 32]
13
14
15# merge_method: linear
16# base_model: Cognitive-Lab/LLama3-Gaja-Hindi-8B-v0.1
17# tokenizer_source: Cognitive-Lab/LLama3-Gaja-Hindi-8B-v0.1
18
19# parameters:
20# t:
21# - filter: self_attn
22# value: 0.05
23# - filter: mlp
24# value: 0.05
25# - filter: norm
26# value: 0.02
27# - filter: embed_tokens
28# value: 0.0
29# - filter: lm_head
30# value: 0.0
31# - value: 0.02
32
33# dtype: bfloat16
34
35# slices:
36# - sources:
37# - model: DeepSeek-R1-0528
38# layer_range: [0, 32]
39# - model: subhrokomol/Llama2-7B-Hindi-finetuned
40# layer_range: [0, 32]
41# merge_method: slerp
42# base_model: DeepSeek-R1-0528
43# tokenizer_source: subhrokomol/Llama2-7B-Hindi-finetuned
44# parameters:
45# t:
46# - filter: self_attn
47# value: [0.7, 0.8, 1.0, 0.8, 0.7]
48# - filter: mlp
49# value: [0.6, 0.5, 0.3, 0.5, 0.6]
50# - value: 0.6
51# dtype: bfloat16
52
53slices:
54 - sources:
55 - model: subhrokomol/Llama2-7B-Hindi-finetuned
56 layer_range: [0, 32]
57 - model: WizardLM/WizardMath-7B-V1.0
58 layer_range: [0, 32]
59merge_method: slerp
60base_model: subhrokomol/Llama2-7B-Hindi-finetuned
61parameters:
62 t:
63 - value: 0.5
64dtype: bfloat16
65
66
67
