Views
No views yet
git installed on your system.pip:python3 -m pip install --upgrade pipaccelerate:python3 -m pip install acceleratebitsandbytespython3 -m pip install bitsandbytesbitsandbytes repository and install it:1git clone https://github.com/TimDettmers/bitsandbytes.git
2cd bitsandbytes
3CUDA_VERSION=118 make cuda11x
4python3 -m pip install .
5cd ..CUDA_VERSION:nvcc --versionFastChat repository and install it:1git clone https://github.com/lm-sys/FastChat.git
2cd FastChat
3python3 -m pip install -e .
4cd ..git-lfs:1curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
2sudo apt-get install git-lfs
3git lfs installvicuna-7b model:git clone https://huggingface.co/helloollel/vicuna-7bpython3 -m fastchat.serve.cli --model-path ./vicuna-7bvicuna-7b model.1import argparse
2import time
3
4import torch
5from transformers import AutoTokenizer, AutoModelForCausalLM, LlamaTokenizer
6
7from fastchat.conversation import conv_templates, SeparatorStyle
8from fastchat.serve.monkey_patch_non_inplace import replace_llama_attn_with_non_inplace_operations
9
10
11def load_model(model_name, device, num_gpus, load_8bit=False):
12 if device == "cpu":
13 kwargs = {}
14 elif device == "cuda":
15 kwargs = {"torch_dtype": torch.float16}
16 if load_8bit:
17 if num_gpus != "auto" and int(num_gpus) != 1:
18 print("8-bit weights are not supported on multiple GPUs. Revert to use one GPU.")
19 kwargs.update({"load_in_8bit": True, "device_map": "auto"})
20 else:
21 if num_gpus == "auto":
22 kwargs["device_map"] = "auto"
23 else:
24 num_gpus = int(num_gpus)
25 if num_gpus != 1:
26 kwargs.update({
27 "device_map": "auto",
28 "max_memory": {i: "13GiB" for i in range(num_gpus)},
29 })
30 elif device == "mps":
31 # Avoid bugs in mps backend by not using in-place operations.
32 kwargs = {"torch_dtype": torch.float16}
33 replace_llama_attn_with_non_inplace_operations()
34 else:
35 raise ValueError(f"Invalid device: {device}")
36
37 tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)
38 model = AutoModelForCausalLM.from_pretrained(model_name,
39 low_cpu_mem_usage=True, **kwargs)
40
41 # calling model.cuda() mess up weights if loading 8-bit weights
42 if device == "cuda" and num_gpus == 1 and not load_8bit:
43 model.to("cuda")
44 elif device == "mps":
45 model.to("mps")
46
47 return model, tokenizer
48
49
50@torch.inference_mode()
51def generate_stream(tokenizer, model, params, device,
52 context_len=2048, stream_interval=2):
53 """Adapted from fastchat/serve/model_worker.py::generate_stream"""
54
55 prompt = params["prompt"]
56 l_prompt = len(prompt)
57 temperature = float(params.get("temperature", 1.0))
58 max_new_tokens = int(params.get("max_new_tokens", 256))
59 stop_str = params.get("stop", None)
60
61 input_ids = tokenizer(prompt).input_ids
62 output_ids = list(input_ids)
63
64 max_src_len = context_len - max_new_tokens - 8
65 input_ids = input_ids[-max_src_len:]
66
67 for i in range(max_new_tokens):
68 if i == 0:
69 out = model(
70 torch.as_tensor([input_ids], device=device), use_cache=True)
71 logits = out.logits
72 past_key_values = out.past_key_values
73 else:
74 attention_mask = torch.ones(
75 1, past_key_values[0][0].shape[-2] + 1, device=device)
76 out = model(input_ids=torch.as_tensor([[token]], device=device),
77 use_cache=True,
78 attention_mask=attention_mask,
79 past_key_values=past_key_values)
80 logits = out.logits
81 past_key_values = out.past_key_values
82
83 last_token_logits = logits[0][-1]
84
85 if device == "mps":
86 # Switch to CPU by avoiding some bugs in mps backend.
87 last_token_logits = last_token_logits.float().to("cpu")
88
89 if temperature < 1e-4:
90 token = int(torch.argmax(last_token_logits))
91 else:
92 probs = torch.softmax(last_token_logits / temperature, dim=-1)
93 token = int(torch.multinomial(probs, num_samples=1))
94
95 output_ids.append(token)
96
97 if token == tokenizer.eos_token_id:
98 stopped = True
99 else:
100 stopped = False
101
102 if i % stream_interval == 0 or i == max_new_tokens - 1 or stopped:
103 output = tokenizer.decode(output_ids, skip_special_tokens=True)
104 pos = output.rfind(stop_str, l_prompt)
105 if pos != -1:
106 output = output[:pos]
107 stopped = True
108 yield output
109
110 if stopped:
111 break
112
113 del past_key_values
114
115args = dict(
116 model_name='./vicuna-7b',
117 device='cuda',
118 num_gpus='1',
119 load_8bit=True,
120 conv_template='vicuna_v1.1',
121 temperature=0.7,
122 max_new_tokens=512,
123 debug=False
124)
125
126args = argparse.Namespace(**args)
127
128model_name = args.model_name
129
130# Model
131model, tokenizer = load_model(args.model_name, args.device,
132 args.num_gpus, args.load_8bit)
133
134# Chat
135conv = conv_templates[args.conv_template].copy()
136
137def chat(inp):
138 conv.append_message(conv.roles[0], inp)
139 conv.append_message(conv.roles[1], None)
140 prompt = conv.get_prompt()
141
142 params = {
143 "model": model_name,
144 "prompt": prompt,
145 "temperature": args.temperature,
146 "max_new_tokens": args.max_new_tokens,
147 "stop": conv.sep if conv.sep_style == SeparatorStyle.SINGLE else conv.sep2,
148 }
149
150 print(f"{conv.roles[1]}: ", end="", flush=True)
151 pre = 0
152 for outputs in generate_stream(tokenizer, model, params, args.device):
153 outputs = outputs[len(prompt) + 1:].strip()
154 outputs = outputs.split(" ")
155 now = len(outputs)
156 if now - 1 > pre:
157 print(" ".join(outputs[pre:now-1]), end=" ", flush=True)
158 pre = now - 1
159 print(" ".join(outputs[pre:]), flush=True)
160
161 conv.messages[-1][-1] = " ".join(outputs)chat("what's the meaning of life?")