A LORA adapter for
meta-llama/Llama-3.3-70B-Instruct-Reference. This model was trained with SFT using
Adaption's AutoScientist on the V5_global_employment_law_qa dataset.
1{
2 "job_id": "146e0588-0b1e-4c32-bc19-77868ef739b2",
3 "training_experiment_id": "756d3205-0c3e-4139-9e8f-84aeed240682",
4 "original_model_name": "meta-llama/Llama-3.3-70B-Instruct-Reference",
5 "trained_model_name": "adaption_global_employment_law_qa",
6 "training_method": "sft",
7 "training_type": "lora",
8 "data_format": "chat",
9 "hyperparams": {
10 "lora": "true",
11 "lora_r": 16,
12 "n_evals": 5,
13 "n_epochs": 2,
14 "batch_size": "max",
15 "lora_alpha": 32,
16 "lora_dropout": 0,
17 "min_lr_ratio": 0.1,
18 "warmup_ratio": 0.05,
19 "weight_decay": 0.05,
20 "learning_rate": 0.0001,
21 "max_grad_norm": 1,
22 "base_model_size": "70B",
23 "train_on_inputs": "false",
24 "training_method": "sft",
25 "lr_scheduler_type": "cosine",
26 "scheduler_num_cycles": 0.5,
27 "lora_trainable_modules": "all-linear"
28 }
29}
The model was trained on 28,863 rows of adapted data with the following domain distribution: legal (45%), hr (15%), math (13%), data-analysis-visualization (10%), writing-editing-communication (8%), science (2%), corporate-business (2%), other (1%), geography (1%), agriculture (1%), history (1%), code (1%), language (0%), finance (0%), transportation (0%), personal-finance (0%), product-advice (0%), academic-education (0%), how-to (0%), fitness-sports (0%), architecture-design (0%), news (0%), logic (0%), travel (0%).
The model was evaluated on an in-distribution held-out test set as well as a broader domain-specific test set to measure generalization.
1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer
3from peft import PeftModel
4
5BASE = "meta-llama/Llama-3.3-70B-Instruct-Reference"
6ADAPTER = "<this-repo-id>"
7
8device = "cuda" if torch.cuda.is_available() else "cpu"
9dtype = torch.float32 if device == "cpu" else torch.bfloat16
10
11base = AutoModelForCausalLM.from_pretrained(BASE, dtype=dtype).to(device)
12model = PeftModel.from_pretrained(base, ADAPTER)
13# Optional: merge the LoRA weights into the base for faster inference
14model = model.merge_and_unload()
15model.eval()
16
17tokenizer = AutoTokenizer.from_pretrained(BASE)
18messages = [{"role": "user", "content": "Hello!"}]
19text = tokenizer.apply_chat_template(
20 messages, tokenize=False, add_generation_prompt=True)
21inputs = tokenizer(text, return_tensors="pt").to(device)
22
23with torch.inference_mode():
24 out = model.generate(**inputs, max_new_tokens=512)
25print(tokenizer.decode(out[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True))
