Views
No views yet
1import {
2 AutoProcessor,
3 AutoModelForVision2Seq,
4 load_image,
5 TextStreamer,
6} from "@huggingface/transformers";
7
8// Initialize processor and model
9const model_id = "onnx-community/granite-docling-258M-ONNX";
10const processor = await AutoProcessor.from_pretrained(model_id);
11const model = await AutoModelForVision2Seq.from_pretrained(model_id, {
12 dtype: "fp32",
13 // device: "webgpu",
14});
15
16// Load images
17const image1 = await load_image("https://huggingface.co/ibm-granite/granite-docling-258M/resolve/main/assets/new_arxiv.png");
18
19// Create input messages
20const messages = [
21 {
22 role: "user",
23 content: [
24 { type: "image" },
25 { type: "text", text: "Convert this page to docling." },
26 ],
27 },
28];
29
30// Prepare inputs
31const text = processor.apply_chat_template(messages, { add_generation_prompt: true });
32const inputs = await processor(text, [image1], {
33 // Set `do_image_splitting: true` to split images into multiple patches.
34 // NOTE: This uses more memory, but can provide more accurate results.
35 do_image_splitting: true,
36});
37
38// Generate outputs
39const generated_ids = await model.generate({
40 ...inputs,
41 max_new_tokens: 4096,
42 streamer: new TextStreamer(processor.tokenizer, {
43 skip_prompt: true,
44 skip_special_tokens: false,
45 }),
46});
47const generated_texts = processor.batch_decode(
48 generated_ids.slice(null, [inputs.input_ids.dims.at(-1), null]),
49 { skip_special_tokens: true },
50);
51console.log(generated_texts[0]);1from transformers import AutoConfig, AutoProcessor
2from transformers.image_utils import load_image
3from huggingface_hub import hf_hub_download
4import onnxruntime
5import numpy as np
6
7
8# 1. Load models
9## Load config and processor
10model_id = "onnx-community/granite-docling-258M-ONNX"
11config = AutoConfig.from_pretrained(model_id)
12processor = AutoProcessor.from_pretrained(model_id)
13
14## Download models from the Hugging Face Hub
15vision_model_path = hf_hub_download(model_id, subfolder="onnx", filename="vision_encoder.onnx") # graph
16hf_hub_download(model_id, subfolder="onnx", filename="vision_encoder.onnx_data") # weights
17embed_model_path = hf_hub_download(model_id, subfolder="onnx", filename="embed_tokens.onnx") # graph
18hf_hub_download(model_id, subfolder="onnx", filename="embed_tokens.onnx_data") # weights
19decoder_model_path = hf_hub_download(model_id, subfolder="onnx", filename="decoder_model_merged.onnx") # graph
20hf_hub_download(model_id, subfolder="onnx", filename="decoder_model_merged.onnx_data") # weights
21
22## Load sessions
23vision_session = onnxruntime.InferenceSession(vision_model_path)
24embed_session = onnxruntime.InferenceSession(embed_model_path)
25decoder_session = onnxruntime.InferenceSession(decoder_model_path)
26
27## Set config values
28num_key_value_heads = config.text_config.num_key_value_heads
29head_dim = config.text_config.head_dim
30num_hidden_layers = config.text_config.num_hidden_layers
31eos_token_id = config.text_config.eos_token_id
32image_token_id = config.image_token_id
33
34
35# 2. Prepare inputs
36## Create input messages
37messages = [
38 {
39 "role": "user",
40 "content": [
41 {"type": "image"},
42 {"type": "text", "text": "Convert this page to docling."}
43 ]
44 },
45]
46
47## Load image and apply processor
48image = load_image("https://huggingface.co/ibm-granite/granite-docling-258M/resolve/main/assets/new_arxiv.png")
49prompt = processor.apply_chat_template(messages, add_generation_prompt=True)
50inputs = processor(text=prompt, images=[image], return_tensors="np")
51
52## Prepare decoder inputs
53batch_size = inputs['input_ids'].shape[0]
54past_key_values = {
55 f'past_key_values.{layer}.{kv}': np.zeros([batch_size, num_key_value_heads, 0, head_dim], dtype=np.float32)
56 for layer in range(num_hidden_layers)
57 for kv in ('key', 'value')
58}
59image_features = None
60input_ids = inputs['input_ids']
61attention_mask = inputs['attention_mask']
62
63
64# 3. Generation loop
65max_new_tokens = 4096
66generated_tokens = np.array([[]], dtype=np.int64)
67for i in range(max_new_tokens):
68 inputs_embeds = embed_session.run(None, {'input_ids': input_ids})[0]
69
70 if image_features is None:
71 ## Only compute vision features if not already computed
72 image_features = vision_session.run(None, dict(
73 pixel_values=inputs['pixel_values'],
74 pixel_attention_mask=inputs['pixel_attention_mask'].astype(np.bool_)
75 ))[0]
76
77 ## Merge text and vision embeddings
78 inputs_embeds[inputs['input_ids'] == image_token_id] = image_features.reshape(-1, image_features.shape[-1])
79
80 logits, *present_key_values = decoder_session.run(None, dict(
81 inputs_embeds=inputs_embeds,
82 attention_mask=attention_mask,
83 **past_key_values,
84 ))
85
86 ## Update values for next generation loop
87 input_ids = logits[:, -1].argmax(-1, keepdims=True)
88 attention_mask = np.concatenate([attention_mask, np.ones((batch_size, 1), dtype=attention_mask.dtype)], axis=-1)
89 for j, key in enumerate(past_key_values):
90 past_key_values[key] = present_key_values[j]
91
92 generated_tokens = np.concatenate([generated_tokens, input_ids], axis=-1)
93 if (input_ids == eos_token_id).all():
94 break
95
96 ## (Optional) Streaming
97 print(processor.decode(input_ids[0]), end='')
98print()
99
100
101# 4. Do something with the final output
102print(processor.batch_decode(generated_tokens, skip_special_tokens=False)[0])<doctag><page_header><loc_115><loc_27><loc_385><loc_34>Energy Budget of WASP-121 b from JWST/NIRISS Phase Curve</page_header>
<page_header><loc_454><loc_28><loc_459><loc_34>9</page_header>
<text><loc_41><loc_42><loc_239><loc_88>while the kernel weights are structured as ( N$_{slice}$ , N$_{time}$ ). This precomputation significantly accelerates our calculations, which is essential since the longitudinal slices are at least partially degenerate with one another. Consequently, the fits require more steps and walkers to ensure proper convergence.</text>
<text><loc_41><loc_89><loc_239><loc_206>To address this, we follow a similar approach to our sinusoidal fits using emcee , but we increase the total number of steps to 100,000 and use 100 walkers. Na¨ıvely, the fit would include 2 N$_{slice}$ + 1 parameters: N$_{slice}$ for the albedo values, N$_{slice}$ for the emission parameters, and one additional scatter parameter, σ . However, since night-side slices do not contribute to the reflected light component, we exclude these albedo values from the fit. In any case, our choice of 100 walkers ensures a sufficient number of walkers per free parameter. Following Coulombe et al. (2025) we set an upper prior limit of 3 / 2 on all albedo slices as a fully Lambertian sphere ( A$_{i}$ = 1 ) corresponds to a geometric albedo of A$_{g}$ = 2 / 3. For thermal emission we impose a uniform prior between 0 and 500 ppm for each slice.</text>
<text><loc_41><loc_207><loc_239><loc_269>We choose to fit our detrended lightcurves considering 4, 6 and 8 longitudinal slices ( N$_{slice}$ = 4 , 6 , 8). However, we show the results of the simplest 4 slice model. As in our previous fits, we conduct an initial run with 25,000 steps (25% of the total run) and use the maximumprobability parameters from this preliminary fit as the starting positions for the final 75,000-step run. We then discard the first 60% of the final run as burn-in.</text>
<section_header_level_1><loc_73><loc_276><loc_207><loc_283>2.5. Planetary Effective Temperature</section_header_level_1>
<text><loc_41><loc_286><loc_239><loc_348>Phase curves are the only way to probe thermal emission from the day and nightside of an exoplanet and hence determine its global energy budget (Partier & Crossfield 2018). The wavelength range of NIRISS/SOSS covers a large portion of the emitted flux of WASP-121 b ( ∼ 50-83%; see Figure 2), enabling a precise and robust constraint of the planet's energy budget.</text>
<text><loc_41><loc_349><loc_239><loc_364>We convert the fitted F$_{p}$ / F$_{∗}$ emission spectra to brightness temperature by wavelength,</text>
<formula><loc_60><loc_368><loc_238><loc_387>T _ { b r i g h t } = \frac { h c } { k \lambda } \cdot \left [ \ln \left ( \frac { 2 b c ^ { 2 } } { \lambda ^ { 5 } B _ { \lambda , p l a n e t } } + 1 \right ) \right ] ^ { - 1 } ,</formula>
<text><loc_41><loc_391><loc_178><loc_398>where the planet's thermal emission is</text>
<formula><loc_84><loc_403><loc_238><loc_419>B _ { \lambda , \text {planet} } = \frac { F _ { p } / F _ { * } } { ( R _ { p } / R _ { * } ) ^ { 2 } } \cdot B _ { \lambda , \text {star} } \, .</formula>
<text><loc_41><loc_425><loc_239><loc_455>There are many ways of converting brightness temperatures to effective temperature, including the ErrorWeighted Mean (EWM), Power-Weighted mean (PWM) and with a Gaussian Process (Schwartz & Cowan 2015;</text>
<chart><loc_273><loc_49><loc_454><loc_134><line_chart><caption><loc_261><loc_141><loc_459><loc_264>Figure 2. Estimated captured flux of the planet assuming the planet radiates as a blackbody. The captured flux is calculated as the ratio of the integrated blackbody emission within the instrument's band pass to the total emission over all wavelengths, i.e., γ = ∫ λ$_{max}$ λ$_{min}$ B ( λ, T ) dλ/ ∫ ∞ 0 B ( λ, T ) dλ . The captured flux fraction is shown for NIRISS SOSS [0.6-2.85 µ m] (red line); Hubble WFC3 [1.12-1.64 µ m] (dashed green line); NIRSpec G395H [2.7-5.15 µ m] (dash dotted blue line). The red-shaded region shows the temperature range on WASP-121 b based on our T$_{eff}$ estimates. Red dashed lines indicate the boundaries of the planet's temperature range within the NIRISS SOSS captured flux fraction. From this we estimate that these observations capture between 55% and 82% of the planet's bolometric flux, depending on orbital phase. Using the minimum temperature from the NAMELESS fit, this estimate decreases to 50%. In either case, the wavelength coverage of NIRISS exceeds that of any other instrument.</caption></chart>
<text><loc_261><loc_273><loc_459><loc_359>Pass et al. 2019). In this work, we elect to compute our effective temperature estimates with a novel method that is essentially a combination of the PWM and EWM. We create the effective temperature by using a simple Monte Carlo process. First, we perturb our F$_{p}$ / F$_{s}$ emission spectra at each point in the orbit by a Gaussian based on the measurement uncertainty. Our new emission spectrum is then used to create an estimate of the brightness temperature spectrum. This process is repeated at each orbital phase. We then estimate the effective temperature, T$_{eff}$ for a given orbital phase as</text>
<formula><loc_317><loc_362><loc_459><loc_382>T _ { \text {eff} } = \frac { \sum _ { i = 1 } ^ { N } w _ { i } T _ { \text {bright} , i } } { \sum _ { i = 1 } ^ { N } w _ { i } } ,</formula>
<text><loc_261><loc_384><loc_459><loc_414>where w$_{i}$ is the weight for the i -th wavelength given by the fraction of the planet's bolometric flux that falls within that wavelength bin scaled by the inverse variance of the measurement,</text>
<formula><loc_305><loc_417><loc_459><loc_437>w _ { i } = \frac { \int _ { \lambda _ { i } } ^ { \lambda _ { i } + 1 } B ( \lambda _ { i } , T _ { \text {est} } ) \, d \lambda } { \int _ { 0 } ^ { \infty } B ( \lambda _ { i } , T _ { \text {est} } ) \, d \lambda } \cdot \frac { 1 } { \sigma _ { i } ^ { 2 } } ,</formula>
<text><loc_261><loc_440><loc_459><loc_454>with T$_{est}$ representing an estimated effective temperature at the orbital phase of interest. When computing</text>
</doctag><|end_of_text|>