Views
No views yet
Qwen/Qwen1.5-4B with LoRA fine-tuning on:ai-factory/red_pajama_subset_arxiv_subset: ArXiv papers truncated to 4096-token chunks.ai-factory/red_pajama_subset_stackexchange_subset: Samples are formatted using a chat template with two roles: user (representing questions) and other (representing answers).ai-factory/glaiveai-reasoning-v1-20m-chat: Samples are formatted using a chat template with two roles: user (representing questions) and me (representing the AI).1from transformers import AutoTokenizer, AutoModelForCausalLM
2model = AutoModelForCausalLM.from_pretrained("your-hf-username/full_finetuned_qwen4b")
3tokenizer = AutoTokenizer.from_pretrained("ai-factory/giant")
4# Load tokenizer
5tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME, trust_remote_code=True)
6tokenizer.pad_token = tokenizer.eos_token if tokenizer.pad_token is None else tokenizer.pad_token
7
8# Load base model
9base_model = AutoModelForCausalLM.from_pretrained(
10 BASE_MODEL,
11 torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
12 device_map="auto",
13 trust_remote_code=True,
14 use_safetensors=True,
15 local_files_only=True
16)
17
18# Apply LoRA
19peft_config = LoraConfig(
20 task_type=TaskType.CAUSAL_LM,
21 r=8,
22 lora_alpha=32,
23 lora_dropout=0.05,
24 bias="none",
25 target_modules=["q_proj", "k_proj", "v_proj", "o_proj"]
26)
27model = get_peft_model(base_model, peft_config)
28model.eval()
29if torch.cuda.is_available():
30 model = model.cuda()
31
32# Load streaming datasets
33arxiv = load_dataset("ai-factory/red_pajama_subset_arxiv_subset", split="train", streaming=True)
34glaive = load_dataset("ai-factory/glaiveai-reasoning-v1-20m-chat", split="train", streaming=True)
35stack = load_dataset("ai-factory/red_pajama_subset_stackexchange_subset", split="train", streaming=True)
36
37def tokenize(example):
38 return tokenizer(example["text"], truncation=True, max_length=4096)
39
40# Tokenize small samples
41tokenized_arxiv = map(tokenize, islice(arxiv, args.sample_size))
42tokenized_glaive = map(tokenize, islice(glaive, args.sample_size))
43tokenized_stack = map(tokenize, islice(stack, args.sample_size))
44
45# Run forward + backward pass (init LoRA weights)
46print("🔥 Training one step to initialize LoRA...")
47for i, sample in enumerate(tokenized_arxiv):
48 if not sample.get("input_ids"):
49 continue
50 ids = torch.tensor(sample["input_ids"]).unsqueeze(0).to(model.device)
51 labels = ids.clone()
52 loss = model(input_ids=ids, labels=labels).loss
53 loss.backward()
54 break
55
56# Merge LoRA and save
57print("🔁 Merging adapter into base model...")
58merged_model = model.merge_and_unload()
59merged_model.save_pretrained(SAVE_DIR, safe_serialization=True)
60tokenizer.save_pretrained(SAVE_DIR)
61print(f"✅ Merged model saved to /mnt/i/sub80/merged/3/ai_factory/full_finetuned_qwen4b")