Views
No views yet
1from transformers import AutoTokenizer, AutoModelForCausalLM
2import torch
3
4model_id = "mzbac/llama-3-8B-Instruct-function-calling"
5tokenizer = AutoTokenizer.from_pretrained(model_id)
6model = AutoModelForCausalLM.from_pretrained(
7 model_id,
8 torch_dtype=torch.bfloat16,
9 device_map="auto",
10)
11
12tool = {
13 "name": "search_web",
14 "description": "Perform a web search for a given search terms.",
15 "parameter": {
16 "type": "object",
17 "properties": {
18 "search_terms": {
19 "type": "array",
20 "items": {"type": "string"},
21 "description": "The search queries for which the search is performed.",
22 "required": True,
23 }
24 }
25 },
26 }
27
28messages = [
29 {
30 "role": "system",
31 "content": f"You are a helpful assistant with access to the following functions. Use them if required - {str(tool)}",
32 },
33 {"role": "user", "content": "Today's news in Melbourne, just for your information, today is April 27, 2014."},
34 ]
35
36input_ids = tokenizer.apply_chat_template(
37 messages,
38 add_generation_prompt=True,
39 return_tensors="pt"
40).to(model.device)
41
42terminators = [
43 tokenizer.eos_token_id,
44 tokenizer.convert_tokens_to_ids("<|eot_id|>")
45]
46
47outputs = model.generate(
48 input_ids,
49 max_new_tokens=256,
50 eos_token_id=terminators,
51 do_sample=True,
52 temperature=0.1,
53)
54response = outputs[0]
55print(tokenizer.decode(response))
56
57# <|begin_of_text|><|start_header_id|>system<|end_header_id|>
58
59# You are a helpful assistant with access to the following functions. Use them if required - {'name':'search_web', 'description': 'Perform a web search for a given search terms.', 'parameter': {'type': 'object', 'properties': {'search_terms': {'type': 'array', 'items': {'type':'string'}, 'description': 'The search queries for which the search is performed.','required': True}}}}<|eot_id|><|start_header_id|>user<|end_header_id|>
60
61# Today's news in Melbourne, just for your information, today is April 27, 2014.<|eot_id|><|start_header_id|>assistant<|end_header_id|>
62
63# <functioncall> {"name": "search_web", "arguments": '{"search_terms": ["Melbourne news", "April 27, 2014"]}'}<|eot_id|>1# The path to the local model directory or Hugging Face repo.
2model: "meta-llama/Meta-Llama-3-8B-Instruct"
3# Whether or not to train (boolean)
4train: true
5
6# Directory with {train, valid, test}.jsonl files
7data: "data"
8
9# The PRNG seed
10seed: 0
11
12# Number of layers to fine-tune
13lora_layers: 32
14
15# Minibatch size.
16batch_size: 1
17
18# Iterations to train for.
19iters: 6000
20
21# Number of validation batches, -1 uses the entire validation set.
22val_batches: 25
23
24# Adam learning rate.
25learning_rate: 1e-6
26
27# Number of training steps between loss reporting.
28steps_per_report: 10
29
30# Number of training steps between validations.
31steps_per_eval: 200
32
33# Load path to resume training with the given adapter weights.
34resume_adapter_file: null
35
36# Save/load path for the trained adapter weights.
37adapter_path: "adapters"
38
39# Save the model every N iterations.
40save_every: 1000
41
42# Evaluate on the test set after training
43test: false
44
45# Number of test set batches, -1 uses the entire test set.
46test_batches: 100
47
48# Maximum sequence length.
49max_seq_length: 8192
50
51# Use gradient checkpointing to reduce memory use.
52grad_checkpoint: false
53
54# LoRA parameters can only be specified in a config file
55lora_parameters:
56 # The layer keys to apply LoRA to.
57 # These will be applied for the last lora_layers
58 keys: ['mlp.gate_proj', 'mlp.down_proj', 'self_attn.q_proj', 'mlp.up_proj', 'self_attn.o_proj','self_attn.v_proj', 'self_attn.k_proj']
59 rank: 128
60 alpha: 256
61 scale: 10.0
62 dropout: 0.05
63
64# Schedule can only be specified in a config file, uncomment to use.
65#lr_schedule:
66# name: cosine_decay
67# warmup: 100 # 0 for no warmup
68# warmup_init: 1e-7 # 0 if not specified
69# arguments: [1e-6, 1000, 1e-7] # passed to scheduler