Views
No views yet
1import torch
2from peft import LoraConfig, PeftModel
3
4from transformers import (
5 AutoModelForCausalLM,
6 AutoTokenizer,
7 BitsAndBytesConfig,
8 HfArgumentParser,
9 TrainingArguments,
10 TextStreamer,
11 pipeline,
12 logging,
13)
14
15device_map = {"": 0}
16use_4bit = True
17bnb_4bit_compute_dtype = "float16"
18bnb_4bit_quant_type = "nf4"
19use_nested_quant = False
20compute_dtype = getattr(torch, bnb_4bit_compute_dtype)
21
22bnb_config = BitsAndBytesConfig(
23 load_in_4bit=use_4bit,
24 bnb_4bit_quant_type=bnb_4bit_quant_type,
25 bnb_4bit_compute_dtype=compute_dtype,
26 bnb_4bit_use_double_quant=use_nested_quant,
27)
28
29based_model_path = "DavidLanz/Llama3-tw-8B-Instruct"
30adapter_path = "DavidLanz/Llama3_tw_8B_btc_qlora"
31
32base_model = AutoModelForCausalLM.from_pretrained(
33 based_model_path,
34 low_cpu_mem_usage=True,
35 return_dict=True,
36 quantization_config=bnb_config,
37 torch_dtype=torch.float16,
38 device_map=device_map,
39)
40model = PeftModel.from_pretrained(base_model, adapter_path)
41
42tokenizer = AutoTokenizer.from_pretrained(based_model_path, trust_remote_code=True)
43
44import torch
45from transformers import pipeline, TextStreamer
46
47text_gen_pipeline = pipeline(
48 "text-generation",
49 model=model,
50 model_kwargs={"torch_dtype": torch.bfloat16},
51 tokenizer=tokenizer,
52)
53
54messages = [
55 {
56 "role": "system",
57 "content": "你是一位專業的BTC虛擬貨幣分析師",
58 },
59 {"role": "user", "content": "昨日開盤價為64437.18，最高價為64960.37，最低價為62953.90，收盤價為64808.35，交易量為808273.27。請預測今日BTC的收盤價?"},
60]
61
62prompt = text_gen_pipeline.tokenizer.apply_chat_template(
63 messages,
64 tokenize=False,
65 add_generation_prompt=True
66)
67
68terminators = [
69 text_gen_pipeline.tokenizer.eos_token_id,
70 text_gen_pipeline.tokenizer.convert_tokens_to_ids("<|eot_id|>")
71]
72
73outputs = text_gen_pipeline(
74 prompt,
75 max_new_tokens=256,
76 eos_token_id=terminators,
77 do_sample=True,
78 temperature=0.6,
79 top_p=0.9,
80)
81print(outputs[0]["generated_text"][len(prompt):])