Views
No views yet
| Name | Quant method | Size |
|---|---|---|
| Tinyllama-2B.Q2_K.gguf | Q2_K | 0.73GB |
| Tinyllama-2B.IQ3_XS.gguf | IQ3_XS | 0.81GB |
| Tinyllama-2B.IQ3_S.gguf | IQ3_S | 0.86GB |
| Tinyllama-2B.Q3_K_S.gguf | Q3_K_S | 0.85GB |
| Tinyllama-2B.IQ3_M.gguf | IQ3_M | 0.89GB |
| Tinyllama-2B.Q3_K.gguf | Q3_K | 0.54GB |
| Tinyllama-2B.Q3_K_M.gguf | Q3_K_M | 0.94GB |
| Tinyllama-2B.Q3_K_L.gguf | Q3_K_L | 1.02GB |
| Tinyllama-2B.IQ4_XS.gguf | IQ4_XS | 1.05GB |
| Tinyllama-2B.Q4_0.gguf | Q4_0 | 1.1GB |
| Tinyllama-2B.IQ4_NL.gguf | IQ4_NL | 0.67GB |
| Tinyllama-2B.Q4_K_S.gguf | Q4_K_S | 1.11GB |
| Tinyllama-2B.Q4_K.gguf | Q4_K | 1.16GB |
| Tinyllama-2B.Q4_K_M.gguf | Q4_K_M | 0.42GB |
| Tinyllama-2B.Q4_1.gguf | Q4_1 | 0.13GB |
| Tinyllama-2B.Q5_0.gguf | Q5_0 | 0.06GB |
| Tinyllama-2B.Q5_K_S.gguf | Q5_K_S | 0.0GB |
| Tinyllama-2B.Q5_K.gguf | Q5_K | 0.18GB |
| Tinyllama-2B.Q5_K_M.gguf | Q5_K_M | 0.07GB |
| Tinyllama-2B.Q5_1.gguf | Q5_1 | 0.1GB |
| Tinyllama-2B.Q6_K.gguf | Q6_K | 0.1GB |
| Tinyllama-2B.Q8_0.gguf | Q8_0 | 2.05GB |
### Instruction:
{prompt}
### Response:1dtype: bfloat16
2merge_method: passthrough
3slices:
4- sources:
5 - layer_range: [0, 16]
6 model: concedo/KobbleTinyV2-1.1B
7- sources:
8 - layer_range: [5, 16]
9 model: concedo/KobbleTinyV2-1.1B
10 parameters:
11 scale:
12 - filter: o_proj
13 value: 0.0
14 - filter: down_proj
15 value: 0.0
16 - value: 1.0
17- sources:
18 - layer_range: [5, 16]
19 model: concedo/KobbleTinyV2-1.1B
20 parameters:
21 scale:
22 - filter: o_proj
23 value: 0.0
24 - filter: down_proj
25 value: 0.0
26 - value: 1.0
27- sources:
28 - layer_range: [16, 22]
29 model: concedo/KobbleTinyV2-1.1B1base_model: Fischerboot/2b-tiny-llama-alpaca-instr
2model_type: LlamaForCausalLM
3tokenizer_type: LlamaTokenizer
4
5load_in_8bit: false
6load_in_4bit: true
7strict: false
8
9datasets:
10 - path: Fischerboot/freedom-rp-alpaca-shortend
11 type: alpaca
12 - path: diffnamehard/toxic-dpo-v0.1-NoWarning-alpaca
13 type: alpaca
14 - path: Fischerboot/alpaca-undensored-fixed-50k
15 type: alpaca
16 - path: Fischerboot/DAN-alpaca
17 type: alpaca
18 - path: Fischerboot/rp-alpaca-next-oone
19 type: alpaca
20
21dataset_prepared_path:
22val_set_size: 0.05
23output_dir: ./outputs/24r
24
25adapter: qlora
26lora_model_dir:
27
28sequence_len: 2048
29sample_packing: true
30eval_sample_packing: false
31pad_to_sequence_len: true
32
33lora_r: 32
34lora_alpha: 16
35lora_dropout: 0.05
36lora_target_modules:
37lora_target_linear: true
38lora_fan_in_fan_out:
39
40wandb_project:
41wandb_entity:
42wandb_watch:
43wandb_name:
44wandb_log_model:
45
46gradient_accumulation_steps: 4
47micro_batch_size: 2
48num_epochs: 4
49optimizer: paged_adamw_32bit
50lr_scheduler: cosine
51learning_rate: 0.0002
52
53train_on_inputs: false
54group_by_length: false
55bf16: auto
56fp16:
57tf32: false
58
59gradient_checkpointing: true
60early_stopping_patience:
61resume_from_checkpoint:
62local_rank:
63logging_steps: 1
64xformers_attention: true
65flash_attention: true
66
67warmup_steps: 10
68evals_per_epoch: 2
69saves_per_epoch: 1
70debug:
71deepspeed:
72weight_decay: 0.0
73fsdp:
74fsdp_config:
75special_tokens:| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 1.7881 | 0.0017 | 1 | 2.5329 |
| 1.6899 | 0.4996 | 287 | 1.9272 |
| 1.5511 | 0.9991 | 574 | 1.8750 |
| 1.4797 | 1.4861 | 861 | 1.8476 |
| 1.5279 | 1.9856 | 1148 | 1.8270 |
| 1.4583 | 2.4726 | 1435 | 1.8275 |
| 1.5044 | 2.9721 | 1722 | 1.8215 |
| 1.3051 | 3.4582 | 2009 | 1.8243 |
| 1.5619 | 3.9578 | 2296 | 1.8245 |