Views
No views yet
1MODEL_SPECS = {
2 "architecture": "Decoder-only Transformer",
3 "params": "10B",
4 "context_length": 4096,
5 "hidden_size": 4096,
6 "attention_heads": 32,
7 "kv_heads": 8,
8 "intermediate_size": 14336,
9 "num_layers": 48,
10 "vocab_size": 32000,
11 "position_encoding": "Rotary",
12 "activation": "SwiGLU",
13 "norm_type": "RMSNorm"
14}1from dataclasses import dataclass
2from typing import Optional, List, Dict, Union
3import torch
4import torch.nn.functional as F
5from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
6
7@dataclass
8class GenerationConfig:
9 temperature: float = 0.7
10 top_p: float = 0.9
11 top_k: int = 50
12 repetition_penalty: float = 1.1
13 max_new_tokens: int = 512
14 do_sample: bool = True
15 num_beams: int = 1
16
17class HiberMultiPipeline:
18 def __init__(
19 self,
20 model_name: str = "Hiber-Multi-10B-Instruct",
21 device_map: str = "auto",
22 torch_dtype: Optional[torch.dtype] = torch.bfloat16,
23 load_in_8bit: bool = False,
24 load_in_4bit: bool = False,
25 ):
26 self.config = AutoConfig.from_pretrained(model_name)
27 self.tokenizer = AutoTokenizer.from_pretrained(
28 model_name,
29 padding_side="left",
30 truncation_side="left",
31 )
32
33 quantization_config = None
34 if load_in_8bit or load_in_4bit:
35 from transformers import BitsAndBytesConfig
36 quantization_config = BitsAndBytesConfig(
37 load_in_8bit=load_in_8bit,
38 load_in_4bit=load_in_4bit,
39 bnb_4bit_compute_dtype=torch.bfloat16,
40 bnb_4bit_quant_type="nf4",
41 )
42
43 self.model = AutoModelForCausalLM.from_pretrained(
44 model_name,
45 device_map=device_map,
46 torch_dtype=torch_dtype,
47 quantization_config=quantization_config,
48 trust_remote_code=True,
49 )
50
51 def generate(
52 self,
53 messages: List[Dict[str, str]],
54 generation_config: Optional[GenerationConfig] = None,
55 ) -> str:
56 if generation_config is None:
57 generation_config = GenerationConfig()
58
59 prompt = self.tokenizer.apply_chat_template(
60 messages,
61 tokenize=False,
62 add_generation_prompt=True
63 )
64
65 inputs = self.tokenizer(
66 prompt,
67 return_tensors="pt",
68 padding=True,
69 truncation=True,
70 max_length=self.config.max_position_embeddings,
71 ).to(self.model.device)
72
73 with torch.inference_mode():
74 outputs = self.model.generate(
75 **inputs,
76 pad_token_id=self.tokenizer.pad_token_id,
77 bos_token_id=self.tokenizer.bos_token_id,
78 eos_token_id=self.tokenizer.eos_token_id,
79 **asdict(generation_config),
80 )
81
82 response = self.tokenizer.decode(
83 outputs[0][inputs["input_ids"].shape[1]:],
84 skip_special_tokens=True,
85 )
86 return response.strip()
87
88 @torch.inference_mode()
89 def batch_generate(
90 self,
91 batch_messages: List[List[Dict[str, str]]],
92 generation_config: Optional[GenerationConfig] = None,
93 batch_size: int = 8,
94 ) -> List[str]:
95 responses = []
96 for i in range(0, len(batch_messages), batch_size):
97 batch = batch_messages[i:i + batch_size]
98 responses.extend([
99 self.generate(msgs, generation_config)
100 for msgs in batch
101 ])
102 return responses1LATENCY_PROFILE = {
2 "first_token": 42,
3 "token_throughput": {
4 "batch_1": 31.25,
5 "batch_8": 5.56,
6 "batch_32": 2.38
7 },
8 "context_scaling": {
9 "1024_tokens": 1.0,
10 "2048_tokens": 1.2,
11 "4096_tokens": 1.8
12 }
13}1@software{hiber_multi_2024,
2 title = {Hiber-Multi-10B-Instruct: Advanced Multilingual Language Model},
3 author = {{Hibernates + UCLA Research Team}},
4 year = {2024},
5 publisher = {HuggingFace},
6 version = {1.0.0},
7 architecture = {Transformer},
8 parameters = {10B},
9 license = {LLaMA 3.1}
10}