Views
No views yet
1from transformers import AutoTokenizer, AutoModelForCausalLM
2import torch
3
4model_id = "mzbac/Phi-3-mini-4k-instruct-function-calling"
5
6tokenizer = AutoTokenizer.from_pretrained(model_id)
7model = AutoModelForCausalLM.from_pretrained(
8 model_id,
9 torch_dtype=torch.bfloat16,
10 device_map="auto",
11)
12
13tool = {
14 "name": "search_web",
15 "description": "Perform a web search for a given search terms.",
16 "parameter": {
17 "type": "object",
18 "properties": {
19 "search_terms": {
20 "type": "array",
21 "items": {"type": "string"},
22 "description": "The search queries for which the search is performed.",
23 "required": True,
24 }
25 },
26 },
27}
28
29messages = [
30 {
31 "role": "user",
32 "content": f"You are a helpful assistant with access to the following functions. Use them if required - {str(tool)}",
33 },
34 {"role": "user", "content": "Any news in Melbourne today, May 7, 2024?"},
35]
36
37input_ids = tokenizer.apply_chat_template(
38 messages, add_generation_prompt=True, return_tensors="pt"
39).to(model.device)
40
41terminators = [tokenizer.eos_token_id, tokenizer.convert_tokens_to_ids("<|end|>")]
42
43outputs = model.generate(
44 input_ids,
45 max_new_tokens=256,
46 eos_token_id=terminators,
47 do_sample=True,
48 temperature=0.1,
49)
50response = outputs[0]
51print(tokenizer.decode(response))
52
53# <s><|user|> You are a helpful assistant with access to the following functions. Use them if required - {'name': 'search_web', 'description': 'Perform a web search for a given search terms.', 'parameter': {'type': 'object', 'properties': {'search_terms': {'type': 'array', 'items': {'type': 'string'}, 'description': 'The search queries for which the search is performed.', 'required': True}}}}<|end|><|assistant|>
54# <|user|> Any news in Melbourne today, May 7, 2024?<|end|>
55# <|assistant|> <functioncall> {"name": "search_web", "arguments": {"search_terms": ["news", "Melbourne", "May 7, 2024"]}}<|end|>1# The path to the local model directory or Hugging Face repo.
2model: "microsoft/Phi-3-mini-4k-instruct"
3# Whether or not to train (boolean)
4train: true
5
6# Directory with {train, valid, test}.jsonl files
7data: "data"
8
9# The PRNG seed
10seed: 0
11
12# Number of layers to fine-tune
13lora_layers: 32
14
15# Minibatch size.
16batch_size: 1
17
18# Iterations to train for.
19iters: 111000
20
21# Number of validation batches, -1 uses the entire validation set.
22val_batches: -1
23
24# Adam learning rate.
25learning_rate: 1e-6
26
27# Number of training steps between loss reporting.
28steps_per_report: 10
29
30# Number of training steps between validations.
31steps_per_eval: 200
32
33# Load path to resume training with the given adapter weights.
34# resume_adapter_file: "adapters/adapters.safetensors"
35
36# Save/load path for the trained adapter weights.
37adapter_path: "adapters"
38
39# Save the model every N iterations.
40save_every: 1000
41
42# Evaluate on the test set after training
43test: false
44
45# Number of test set batches, -1 uses the entire test set.
46test_batches: 100
47
48# Maximum sequence length.
49max_seq_length: 4096
50
51# Use gradient checkpointing to reduce memory use.
52grad_checkpoint: false
53
54# LoRA parameters can only be specified in a config file
55lora_parameters:
56 # The layer keys to apply LoRA to.
57 # These will be applied for the last lora_layers
58 keys: ['mlp.down_proj','mlp.gate_up_proj','self_attn.qkv_proj','self_attn.o_proj']
59 rank: 128
60 alpha: 256
61 scale: 10.0
62 dropout: 0.05