This is a merge of pre-trained language models created using
mergekit.
1base_model:
2 model:
3 path: NickyNicky/TinyDolphin-2.8-1.1b_oasst2_chatML_Cluster_1_V1
4dtype: bfloat16
5merge_method: dare_ties
6slices:
7- sources:
8 - layer_range: [0, 22]
9 model:
10 model:
11 path: NickyNicky/TinyDolphin-2.8-1.1b_oasst2_chatML_Cluster_1_V1
12 - layer_range: [0, 22]
13 model:
14 model:
15 path: NickyNicky/TinyDolphin-2.8-1.1b_oasst2_chatML_Cluster_1_V1
16 parameters:
17 density: 0.55
18 weight: 0.55
19 - layer_range: [0, 22]
20 model:
21 model:
22 path: NickyNicky/TinyDolphin-2.8-1.1b_oasst2_chatML_Cluster_2_V1
23 parameters:
24 density: 0.55
25 weight: 0.56
26 - layer_range: [0, 22]
27 model:
28 model:
29 path: NickyNicky/TinyDolphin-2.8-1.1b_oasst2_chatML_Cluster_3_V1
30 parameters:
31 density: 0.55
32 weight: 0.56
1from transformers import (
2 AutoModelForCausalLM,
3 AutoTokenizer,
4 BitsAndBytesConfig,
5 HfArgumentParser,
6 TrainingArguments,
7 pipeline,
8 logging,
9 GenerationConfig,
10 TextIteratorStreamer,
11)
12import torch
13
14new_model= "NickyNicky/TinyDolphin-2.8-1.1b_oasst2_chatML_all_Cluster_merge_v1"
15model = AutoModelForCausalLM.from_pretrained(#f'NickyNicky/{new_model}',
16 new_model,
17 device_map="auto",
18 trust_remote_code=True,
19 torch_dtype=torch.bfloat16,
20
21 low_cpu_mem_usage= True,
22 # use_flash_attention_2=False,
23
24 )
25
26
27tokenizer = AutoTokenizer.from_pretrained(new_model,
28 max_length=2048,
29 trust_remote_code=True,
30 use_fast = True,
31 )
32
33tokenizer.pad_token = tokenizer.eos_token
34# tokenizer.padding_side = 'left'
35tokenizer.padding_side = 'right'
36
37
38prompt= """<|im_start|>system
39You are a helpful AI assistant.<|im_end|>
40<|im_start|>user
41escribe una historia de amor.<|im_end|>
42<|im_start|>assistant
43"""
44
45inputs = tokenizer.encode(prompt,
46 return_tensors="pt",
47 add_special_tokens=False).cuda()#.to("cuda") # False # True
48
49
50generation_config = GenerationConfig(
51 max_new_tokens=700,
52 temperature=0.5,
53 top_p=0.9,
54 top_k=40,
55 repetition_penalty=1.1, #1.1, # 1.0 means no penalty, > 1.0 means penalty, 1.2 from CTRL paper
56 do_sample=True,
57 pad_token_id=tokenizer.eos_token_id,
58 eos_token_id=tokenizer.eos_token_id,
59 )
60outputs = model.generate(
61 generation_config=generation_config,
62 input_ids=inputs,)
63# tokenizer.decode(outputs[0], skip_special_tokens=False) #True
64print(tokenizer.decode(outputs[0], skip_special_tokens=False))
