Views
No views yet
1# Bio-Hacker Chimera Merge Configuration
2# SLERP merge of Biomedical + Cybersecurity Llama-3 models
3#
4# The parameter oscillation creates a gradient through layers:
5# - Early layers: Bio-dominant attention, Cyber-dominant MLP
6# - Middle layers: Balanced mix
7# - Late layers: Cyber-dominant attention, Bio-dominant MLP
8#
9# This should create interesting cross-domain "thinking patterns"
10
11slices:
12 - sources:
13 - model: aaditya/Llama3-OpenBioLLM-8B
14 layer_range: [0, 32]
15 - model: WhiteRabbitNeo/Llama-3-WhiteRabbitNeo-8B-v2.0
16 layer_range: [0, 32]
17
18merge_method: slerp
19
20base_model: aaditya/Llama3-OpenBioLLM-8B
21
22parameters:
23 t:
24 # Self-attention: Start Bio (0), end Cyber (1)
25 # Controls "what to pay attention to"
26 - filter: self_attn
27 value: [0, 0.5, 0.3, 0.7, 1]
28 # MLP: Start Cyber (1), end Bio (0)
29 # Controls "how to transform information"
30 - filter: mlp
31 value: [1, 0.5, 0.7, 0.3, 0]
32 # Default for anything else (embeddings, norms, etc)
33 - value: 0.5
34
35dtype: float16
36