KarantaOCR is fine-tuned from
Qwen/Qwen2.5-VL-3B-Instruct, a vision-language model that combines a strong vision encoder with a large language model.
Through targeted curriculum fine-tuning, KarantaOCR extends these capabilities to robust document understanding across diverse PDF formats and multilingual settings.
-
50,000 PDFs containing text in 10 African languages, crawled from the web
-
Domains include:
- Religious texts
- Legal documents
- Dictionaries
- Novels
- Other long-form and structured documents
-
High-accuracy text extraction from PDFs
-
Table extraction and structured document understanding
-
Robust handling of:
- Multi-column layouts
- Headers and footers
- Mixed scanned and digital PDFs
KarantaOCR is evaluated on the OLMOocr benchmark using pass-rate accuracy. Scores are reported as averages across JSONL files with 95% confidence intervals.
In addition to OLMOocr benchmark, we also create a KarantaOCR-Bench, which focuses specifically on testing OCR extraction on special characters and diacritics.
KarantaOCR processes PDF documents by rendering pages into images and combining them with structured prompts for inference.
1import torch
2from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor
3
4def load_model(model_path: str, device_map: str = "auto", dtype: str = "auto"):
5 model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
6 model_path,
7 torch_dtype=getattr(torch, dtype) if dtype != "auto" else "auto",
8 device_map=device_map,
9 )
10 return model
11
12def load_processor(processor_name: str, min_pixels=None, max_pixels=None):
13 if min_pixels and max_pixels:
14 return AutoProcessor.from_pretrained(
15 processor_name, min_pixels=min_pixels, max_pixels=max_pixels
16 )
17 return AutoProcessor.from_pretrained(processor_name)
1from jinja2 import Template
2
3def render_pdf_to_base64png(
4 local_pdf_path: str, page_num: int, target_longest_image_dim: int = 2048
5) -> str:
6 longest_dim = max(get_pdf_media_box_width_height(local_pdf_path, page_num))
7
8 # Convert PDF page to PNG using pdftoppm
9 pdftoppm_result = subprocess.run(
10 [
11 "pdftoppm",
12 "-png",
13 "-f",
14 str(page_num),
15 "-l",
16 str(page_num),
17 "-r",
18 str(
19 target_longest_image_dim * 72 / longest_dim
20 ), # 72 pixels per point is the conversion factor
21 local_pdf_path,
22 ],
23 timeout=120,
24 stdout=subprocess.PIPE,
25 stderr=subprocess.PIPE,
26 )
27 assert pdftoppm_result.returncode == 0, pdftoppm_result.stderr
28 return base64.b64encode(pdftoppm_result.stdout).decode("utf-8")
29
30def build_message(image_url: str, system_prompt: str, page: int = 0):
31 image_base64 = render_pdf_to_base64png(image_url, page, TARGET_IMAGE_DIM)
32
33 prompt = [
34 {
35 "role": "user",
36 "content": [
37 {
38 "type": "text",
39 "text": system_prompt
40 },
41 {
42 "type": "image",
43 "image": f"data:image/png;base64,{image_base64}",
44 },
45 ],
46 }
47 ]
48 return prompt
1from qwen_vl_utils import process_vision_info
2
3def run_inference(model, processor, messages, max_new_tokens=128, device="cuda"):
4 text = processor.apply_chat_template(
5 messages, tokenize=False, add_generation_prompt=True
6 )
7
8 image_inputs, _ = process_vision_info(messages)
9 inputs = processor(
10 text=[text],
11 images=image_inputs,
12 padding=False,
13 return_tensors="pt",
14 ).to(device)
15
16 generated_ids = model.generate(**inputs, max_new_tokens=max_new_tokens)
17 trimmed_ids = [
18 out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
19 ]
20
21 outputs = processor.batch_decode(
22 trimmed_ids,
23 skip_special_tokens=True,
24 clean_up_tokenization_spaces=False,
25 )
26 return outputs[0]
1model = load_model("taresco/KarantaOCR")
2processor = load_processor("taresco/KarantaOCR")
3
4prompt = """Below is the image of one page of a PDF document.
5Just return the plain text representation of this document as if you were reading it naturally.
6Turn equations into a LaTeX representation, and tables into markdown format. Remove the headers and footers, but keep references and footnotes.
7Read any natural handwriting.
8This is likely one page out of several in the document, so be sure to preserve any sentences that come from the previous page, or continue onto the next page, exactly as they are.
9If there is no text at all that you think you should read, you can output null.
10if the document contains diacritics, please include them in the output.
11Do not hallucinate.
12"""
13
14messages = build_message(
15 image_url="example.pdf",
16 system_prompt=prompt,
17 page=0
18)
19
20output_text = run_inference(model, processor, messages)
21print(output_text)
Coming soon ...
