Views
No views yet
npm i @huggingface/transformersonnx-community/paligemma2-3b-ft-docci-448.1import { AutoProcessor, PaliGemmaForConditionalGeneration, load_image } from '@huggingface/transformers';
2
3// Load processor and model
4const model_id = 'onnx-community/paligemma2-3b-ft-docci-448';
5const processor = await AutoProcessor.from_pretrained(model_id);
6const model = await PaliGemmaForConditionalGeneration.from_pretrained(model_id, {
7 dtype: {
8 embed_tokens: 'fp16', // or 'q8'
9 vision_encoder: 'fp16', // or 'q4', 'q8'
10 decoder_model_merged: 'q4', // or 'q4f16'
11 },
12});
13
14// Prepare inputs
15const url = 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg'
16const raw_image = await load_image(url);
17const prompt = '<image>caption en'; // Caption the image in English
18const inputs = await processor(raw_image, prompt);
19
20// Generate a response
21const output = await model.generate({
22 ...inputs,
23 max_new_tokens: 100,
24})
25
26const generated_ids = output.slice(null, [inputs.input_ids.dims[1], null]);
27const answer = processor.batch_decode(
28 generated_ids,
29 { skip_special_tokens: true },
30);
31console.log(answer[0]);
32// A side view of a light blue 1970s Volkswagen Beetle parked on a gray cement road. It is facing to the right. It has a reflection on the side of it. Behind it is a yellow building with a brown double door on the right. It has a white frame around it. Part of a gray cement wall is visible on the far left.onnx).