Views
No views yet
1from transformers import AutoTokenizer, AutoModelForCausalLM
2import torch
3
4model_id = "mzbac/llama-3-8B-Instruct-function-calling-v0.2"
5tokenizer = AutoTokenizer.from_pretrained(model_id)
6model = AutoModelForCausalLM.from_pretrained(
7 model_id,
8 torch_dtype=torch.bfloat16,
9 device_map="auto",
10
11)
12
13tool = {
14 "name": "search_web",
15 "description": "Perform a web search for a given search terms.",
16 "parameter": {
17 "type": "object",
18 "properties": {
19 "search_terms": {
20 "type": "array",
21 "items": {"type": "string"},
22 "description": "The search queries for which the search is performed.",
23 "required": True,
24 }
25 }
26 },
27 }
28
29messages = [
30 {
31 "role": "system",
32 "content": f"You are a helpful assistant with access to the following functions. Use them if required - {str(tool)}",
33 },
34 {"role": "user", "content": "Today's news in Melbourne, just for your information, today is April 27, 2014."},
35 ]
36
37input_ids = tokenizer.apply_chat_template(
38 messages,
39 add_generation_prompt=True,
40 return_tensors="pt"
41).to(model.device)
42
43terminators = [
44 tokenizer.eos_token_id,
45 tokenizer.convert_tokens_to_ids("<|eot_id|>")
46]
47
48outputs = model.generate(
49 input_ids,
50 max_new_tokens=256,
51 eos_token_id=terminators,
52 do_sample=True,
53 temperature=0.1,
54)
55response = outputs[0]
56print(tokenizer.decode(response))
57
58# <|begin_of_text|><|start_header_id|>system<|end_header_id|>
59
60# You are a helpful assistant with access to the following functions. Use them if required - {'name':'search_web', 'description': 'Perform a web search for a given search terms.', 'parameter': {'type': 'object', 'properties': {'search_terms': {'type': 'array', 'items': {'type':'string'}, 'description': 'The search queries for which the search is performed.','required': True}}}}<|eot_id|><|start_header_id|>user<|end_header_id|>
61
62# Today's news in Melbourne, just for your information, today is April 27, 2014.<|eot_id|><|start_header_id|>assistant<|end_header_id|>
63
64# <functioncall> {"name": "search_web", "arguments": {"search_terms": ["Melbourne news", "April 27, 2014"]}}<|eot_id|>