A YOLO11s (small) model fine-tuned on 3 000 synthetic Windows-style UI screenshots to detect interactive UI elements. Designed as a lightweight computer-vision fallback for Windows UI automation agents when native UI Automation APIs fail.
1from local_ui_locator import detect_elements, find_by_text, safe_click_point
2
3# Detect all UI elements in a screenshot
4detections = detect_elements("screenshot.png", conf=0.3)
5for det in detections:
6 print(f"{det.type}: {det.bbox} score={det.score:.2f}")
7
8# Find element by text
9match = find_by_text("screenshot.png", query="Submit")
10if match:
11 x, y = safe_click_point(match.bbox)
12 print(f"Click at ({x}, {y})")
1from ultralytics import YOLO
2
3model = YOLO("best.pt")
4results = model.predict("screenshot.png", conf=0.3)
