Views
No views yet
1deepspeed moellava/train/train_mem.py \
2 --lora_enable True --lora_r 128 --lora_alpha 256 --mm_projector_lr 0.00000125 \
3 --lora_path /kaggle/temp/lora-llavaqwen \
4 --deepspeed ./scripts/zero3.json \
5 --model_name_or_path Qwen/Qwen1.5-1.8B \
6 --version qwen \
7 --data_path /kaggle/temp/vi_llava_train.json \
8 --image_folder /kaggle/input/coco-2017-dataset/coco2017/train2017 \
9 --image_tower google/siglip-base-patch16-256-multilingual \
10 --image_projector_type mlp2x_gelu \
11 --pretrain_mm_mlp_adapter /kaggle/temp/pt-llavaqwen1.5-1.8b/mm_projector.bin \
12 --mm_vision_select_layer -2 \
13 --mm_use_im_start_end False \
14 --mm_use_im_patch_token False \
15 --image_aspect_ratio pad \
16 --group_by_modality_length True \
17 --fp16 True \
18 --output_dir ./checkpoints/ft-lora-llavaqwen1.5-1.8b-complex_reasoning \
19 --num_train_epochs 1 \
20 --per_device_train_batch_size 2 \
21 --per_device_eval_batch_size 4 \
22 --gradient_accumulation_steps 8 \
23 --evaluation_strategy "no" \
24 --save_strategy "steps" \
25 --save_steps 100 \
26 --save_total_limit 1 \
27 --learning_rate 1e-5 \
28 --weight_decay 0. \
29 --warmup_ratio 0 \
30 --lr_scheduler_type "cosine" \
31 --logging_steps 5 \
32 --tf32 False \
33 --model_max_length 1024 \
34 --gradient_checkpointing True \
35 --dataloader_num_workers 4 \
36 --lazy_preprocess True \
37 --report_to wandb \
38 --run_name ft-llava-qwen1.5-1.8b-lora-vista_reasoning-cont \
39 --push_to_hub True1from typing import Optional, List
2class ModelArguments:
3 model_name_or_path: Optional[str] = "facebook/opt-125m"
4 version: Optional[str] = "v0"
5 freeze_backbone: bool = False
6 tune_mm_mlp_adapter: bool = False
7 mm_vision_select_layer: Optional[int] = -1 # default to the last layer
8 pretrain_mm_mlp_adapter: Optional[str] = None
9 mm_use_im_start_end: bool = False
10 mm_use_im_patch_token: bool = True
11 mm_vision_select_feature: Optional[str] = "patch"
12 # ===================================================================
13 image_tower: Optional[str] = 'google/siglip-base-patch16-256-multilingual'
14 video_tower: Optional[str] = None
15 image_projector_type: Optional[str] = 'linear'
16 video_projector_type: Optional[str] = 'linear'
17 video_global_proj: bool = False
18 video_temproal_proj: bool = False
19 video_spatial_proj: bool = False
20 # ===================================================================
21
22 # =============================================================
23 only_lora_ffn: bool = True
24 moe_enable: bool = False
25 train_modules: Optional[List[str]] = None
26 moe_mode: str = "sparse"
27 moe_layers_idx: Optional[List[int]] = None
28 ep_size: int = 1
29 num_experts: Optional[List[int]] = 4
30 top_k_experts: int = 2
31 capacity_factor: float = 1.
32 eval_capacity_factor: float = 2.
33 min_capacity: int = 0
34 use_residual: bool = False
35 router_aux_loss_coef: float = 0.01
36
37class DataArguments:
38 lazy_preprocess: bool = False
39 is_multimodal: bool = False
40 image_aspect_ratio: str = 'pad'
41 # ===================================================================
42 data_path: Optional[List[str]] = None
43 image_folder: Optional[str] = None
44 video_folder: Optional[str] = None
45 num_frames: int = 8
46
47model_args = ModelArguments()
48data_args = DataArguments()
49
50import torch
51from peft import PeftModel
52from moellava.model import LlavaQwen1_5ForCausalLM
53
54model_name_or_path = 'Qwen/Qwen1.5-1.8B'
55lora_path = 'llavaqwen1.5-lora'
56
57model = LlavaQwen1_5ForCausalLM.from_pretrained(
58 model_name_or_path,
59)
60
61model.to(torch.float16)
62model = PeftModel.from_pretrained(model, lora_path)
63model
64
65import transformers
66
67tokenizer = transformers.AutoTokenizer.from_pretrained(
68 model_args.model_name_or_path,
69 model_max_length=1024,
70 padding_side="right",
71 use_fast=False,
72)
73tokenizer.add_special_tokens({'unk_token': '<|extra_0|>'})
74
75model.get_model().initialize_vision_modules(
76 model_args=model_args,
77)
78
79image_tower = model.get_image_tower()
80image_tower.to(dtype=torch.float16)
81
82data_args.image_processor = image_tower.image_processor
83data_args.is_multimodal = True
84
85model.config.image_aspect_ratio = data_args.image_aspect_ratio
86model.config.tokenizer_padding_side = tokenizer.padding_side
87
88model.config.mm_use_im_start_end = data_args.mm_use_im_start_end = model_args.mm_use_im_start_end
89model.config.mm_use_im_patch_token = model_args.mm_use_im_patch_token
90model.initialize_vision_tokenizer(model_args, tokenizer=tokenizer)
91
92merged_model = model.merge_and_unload()
93merged_model.save_pretrained("llava-qwen1.5-1.8b-complex_reasoning-merged")