Fine-tuned
SmolVLM-500M-Instruct for
vision-based navigation assistance for blind and visually impaired users. Developed as a Master's thesis project at Asia Pacific University.
1from transformers import Idefics3ForConditionalGeneration, AutoProcessor
2from PIL import Image
3import torch
4
5# Load model
6model = Idefics3ForConditionalGeneration.from_pretrained(
7 "msaid1976/SmolVLM-Instruct-Navigation-FineTuned",
8 torch_dtype=torch.float16,
9 device_map="auto",
10 trust_remote_code=True
11)
12processor = AutoProcessor.from_pretrained(
13 "msaid1976/SmolVLM-Instruct-Navigation-FineTuned",
14 trust_remote_code=True
15)
16
17# Prepare input
18image = Image.open("scene.jpg")
19messages = [{
20 "role": "user",
21 "content": [
22 {"type": "image"},
23 {"type": "text", "text": "What do you see?"}
24 ]
25}]
26
27# Generate
28prompt = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
29inputs = processor(text=prompt, images=[image], return_tensors="pt")
30inputs = {k: v.to("cuda") for k, v in inputs.items()}
31
32with torch.no_grad():
33 outputs = model.generate(
34 **inputs,
35 max_new_tokens=150,
36 do_sample=False,
37 pad_token_id=processor.tokenizer.eos_token_id,
38 eos_token_id=processor.tokenizer.eos_token_id
39 )
40
41response = processor.decode(outputs[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True)
42print(response)
1messages = [{
2 "role": "user",
3 "content": [
4 {"type": "image"},
5 {"type": "text", "text": "Is there a chair to the left of the table?"}
6 ]
7}]
8# Output: "Yes, there is a chair to the left of the table."
1messages = [{
2 "role": "user",
3 "content": [
4 {"type": "image"},
5 {"type": "text", "text": "Describe the scene in front of me."}
6 ]
7}]
8# Output: "The scene shows a living room with a brown sofa on the left,
9# a wooden coffee table in the center, and a TV on the wall..."
1messages = [{
2 "role": "user",
3 "content": [
4 {"type": "image"},
5 {"type": "text", "text": "What text is on the sign?"}
6 ]
7}]
8# Output: "The sign says 'EXIT' in red letters."
1# 8-bit quantization (reduces to ~2GB VRAM)
2model = Idefics3ForConditionalGeneration.from_pretrained(
3 "msaid1976/SmolVLM-Instruct-Navigation-FineTuned",
4 load_in_8bit=True,
5 device_map="auto"
6)
7
8# Batch processing
9inputs = processor(
10 text=[prompt1, prompt2, prompt3],
11 images=[[img1], [img2], [img3]],
12 return_tensors="pt",
13 padding=True
14)
1@misc{alqahtani2025smolvlm_navigation,
2 author = {Alqahtani, Muhammad Said},
3 title = {SmolVLM Navigation Assistant: Fine-tuned for Blind Navigation},
4 year = {2025},
5 publisher = {HuggingFace},
6 howpublished = {\url{https://huggingface.co/msaid1976/SmolVLM-Instruct-Navigation-FineTuned}}
7}
8
9@mastersthesis{alqahtani2025thesis,
10 author = {Alqahtani, Muhammad Said},
11 title = {An Efficient Multi-Object Detection and Smart Navigation Using Vision Language Models for Visually Impaired},
12 school = {Asia Pacific University of Technology and Innovation},
13 year = {2025},
14 address = {Kuala Lumpur, Malaysia}
15}
Author: Mohammad Mohamed Said Aly Amin
Institution: Asia Pacific University
Issues: Model Discussions
