Views
No views yet
ds4sd/SmolDocling-256M-preview using mlx-vlm version 0.1.18.
Has configuration files adapted for usage with Docling-Snap.
Refer to the original model card for more details on the model.pip install -U mlx-vlm pillow docling-core1# /// script
2# requires-python = ">=3.12"
3# dependencies = [
4# "docling-core",
5# "mlx-vlm",
6# "pillow",
7# ]
8# ///
9from io import BytesIO
10from pathlib import Path
11from urllib.parse import urlparse
12
13import requests
14from PIL import Image
15from docling_core.types.doc import ImageRefMode
16from docling_core.types.doc.document import DocTagsDocument, DoclingDocument
17from mlx_vlm import load, generate
18from mlx_vlm.prompt_utils import apply_chat_template
19from mlx_vlm.utils import load_config, stream_generate
20
21## Settings
22SHOW_IN_BROWSER = True # Export output as HTML and open in webbrowser.
23
24## Load the model
25model_path = "ds4sd/SmolDocling-256M-preview-mlx-bf16"
26model, processor = load(model_path)
27config = load_config(model_path)
28
29## Prepare input
30prompt = "Convert this page to docling."
31
32# image = "https://ibm.biz/docling-page-with-list"
33image = "https://ibm.biz/docling-page-with-table"
34
35# Load image resource
36if urlparse(image).scheme != "": # it is a URL
37 response = requests.get(image, stream=True, timeout=10)
38 response.raise_for_status()
39 pil_image = Image.open(BytesIO(response.content))
40else:
41 pil_image = Image.open(image)
42
43# Apply chat template
44formatted_prompt = apply_chat_template(processor, config, prompt, num_images=1)
45
46## Generate output
47print("DocTags: \n\n")
48
49output = ""
50for token in stream_generate(
51 model, processor, formatted_prompt, [image], max_tokens=4096, verbose=False
52):
53 output += token.text
54 print(token.text, end="")
55 if "</doctag>" in token.text:
56 break
57
58print("\n\n")
59
60# Populate document
61doctags_doc = DocTagsDocument.from_doctags_and_image_pairs([output], [pil_image])
62# create a docling document
63doc = DoclingDocument(name="SampleDocument")
64doc.load_from_doctags(doctags_doc)
65
66## Export as any format
67# Markdown
68print("Markdown: \n\n")
69print(doc.export_to_markdown())
70
71# HTML
72if SHOW_IN_BROWSER:
73 import webbrowser
74
75 out_path = Path("./output.html")
76 doc.save_as_html(out_path, image_mode=ImageRefMode.EMBEDDED)
77 webbrowser.open(f"file:///{str(out_path.resolve())}")
78