Views
No views yet
microsoft/layoutlmv3-base designed for token classification on scanned medical lab reports. It uses BIOES tagging to extract structured entities such as patient name, doctor name, lab name, date, sex, age, and more. The model works in conjunction with OCR results (PaddleOCR v2.6) to handle scanned documents in image format.microsoft/layoutlmv3-base1import streamlit as st
2import torch
3from transformers import LayoutLMv3Processor, LayoutLMv3ForTokenClassification
4from PIL import Image
5from paddleocr import PaddleOCR
6import numpy as np
7import os
8from huggingface_hub import login
9
10# Login to Hugging Face using the environment variable token
11HF_TOKEN = os.environ.get("HF_TOKEN")
12if not HF_TOKEN:
13 st.error("Hugging Face token not found. Please set 'HF_TOKEN' as an environment variable.")
14else:
15 login(HF_TOKEN)
16
17# Set Streamlit page configuration
18st.set_page_config(layout="wide", page_title="Document Entity Extractor")
19
20st.title("📄 Document Entity Extractor (LayoutLMv3 + PaddleOCR)")
21st.markdown("Upload an image (e.g., a scanned document or a form) to extract structured information.")
22
23@st.cache_resource
24def load_paddle_ocr_model():
25 st.info("Initializing PaddleOCR model...")
26 return PaddleOCR(use_angle_cls=True, lang='en', show_log=False)
27
28@st.cache_resource
29def load_layoutlmv3_model_and_processor(model_name_or_path="parthesh111/layoutlmv3-finetune-bioes-new"):
30 st.info(f"Loading LayoutLMv3 model from {model_name_or_path}...")
31 processor = LayoutLMv3Processor.from_pretrained(model_name_or_path, apply_ocr=False)
32 model = LayoutLMv3ForTokenClassification.from_pretrained(model_name_or_path)
33 return model, processor
34
35def normalize_box(box, width, height):
36 return [
37 int(1000 * (box[0] / width)),
38 int(1000 * (box[1] / height)),
39 int(1000 * (box[2] / width)),
40 int(1000 * (box[3] / height)),
41 ]
42
43def generate_non_pii_summary_with_llm(text_data):
44 if not text_data or text_data.strip() == "No 'O' (Outside) labeled text found.":
45 return "No non-PII data available for summarization."
46
47 return (
48 f"Original Non-PII Text:\n\"{text_data}\"\n\n"
49 f"This section would typically contain a summary or analysis generated by an external Large Language Model "
50 f"(e.g., ChatGPT) based on the provided 'O' labeled text. For demonstration purposes, "
51 f"this is a placeholder showing the input text. Implement your actual LLM API call here.")
52
53def run_inference_logic(image: Image.Image, model, processor, ocr_engine):
54 width, height = image.size
55
56 try:
57 paddle_ocr_result = ocr_engine.ocr(np.array(image), cls=True)
58 words = []
59 boxes = []
60 if not paddle_ocr_result or not paddle_ocr_result[0]:
61 st.warning("PaddleOCR did not detect any text in the image.")
62 return {}, "No text detected by OCR."
63
64 for line in paddle_ocr_result[0]:
65 box_coords = line[0]
66 text = line[1][0]
67 x_min = min([point[0] for point in box_coords])
68 y_min = min([point[1] for point in box_coords])
69 x_max = max([point[0] for point in box_coords])
70 y_max = max([point[1] for point in box_coords])
71 words.append(text)
72 boxes.append([x_min, y_min, x_max, y_max])
73
74 except Exception as e:
75 st.error(f"An error occurred during PaddleOCR processing: {e}")
76 return {}, f"Error during OCR: {e}"
77
78 if not words:
79 return {}, "No extractable words found by OCR."
80
81 normalized_boxes = [normalize_box(box, width, height) for box in boxes]
82
83 try:
84 if not words or not normalized_boxes:
85 st.warning("No words or bounding boxes to encode for LayoutLMv3.")
86 return {}, "No valid data for model inference."
87
88 encoding = processor(image, text=words, boxes=normalized_boxes,
89 return_tensors="pt", truncation=True, padding="max_length", max_length=512)
90
91 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
92 model.to(device)
93 for key, val in encoding.items():
94 encoding[key] = val.to(device)
95
96 with torch.no_grad():
97 outputs = model(**encoding)
98
99 predictions = outputs.logits.argmax(-1).squeeze().tolist()
100 word_ids = encoding.word_ids(batch_index=0)
101
102 word_labels = {}
103 for token_idx, word_idx in enumerate(word_ids):
104 if word_idx is not None and token_idx < len(predictions):
105 if word_idx not in word_labels:
106 word_labels[word_idx] = model.config.id2label[predictions[token_idx]]
107
108 all_grouped_segments = {}
109 current_segment_text = ""
110 current_segment_label = ""
111
112 for i in range(len(words)):
113 label = word_labels.get(i, "O")
114 word = words[i]
115
116 if label == "O":
117 effective_segment_type = "O"
118 else:
119 _, _, entity_type = label.partition('-')
120 effective_segment_type = entity_type
121
122 if current_segment_text and effective_segment_type != current_segment_label:
123 if current_segment_label not in all_grouped_segments:
124 all_grouped_segments[current_segment_label] = []
125 all_grouped_segments[current_segment_label].append(current_segment_text.strip())
126 current_segment_text = ""
127 current_segment_label = ""
128
129 if not current_segment_text:
130 current_segment_text = word
131 current_segment_label = effective_segment_type
132 else:
133 current_segment_text += " " + word
134
135 if current_segment_text and current_segment_label:
136 if current_segment_label not in all_grouped_segments:
137 all_grouped_segments[current_segment_label] = []
138 all_grouped_segments[current_segment_label].append(current_segment_text.strip())
139
140 non_pii_segments = all_grouped_segments.pop("O", [])
141 pii_data = all_grouped_segments
142 non_pii_data_string = " ".join(non_pii_segments) if non_pii_segments else "No 'O' (Outside) labeled text found."
143 return pii_data, non_pii_data_string
144
145 except Exception as e:
146 st.error(f"An error occurred during LayoutLMv3 model inference: {e}")
147 return {}, f"Error during model inference: {e}"
148
149# --- UI ---
150
151uploaded_file = st.file_uploader("Upload an image (JPG, JPEG, PNG)", type=["jpg", "jpeg", "png"])
152
153if uploaded_file is not None:
154 image = Image.open(uploaded_file).convert("RGB")
155
156 with st.spinner("Loading models and processing..."):
157 ocr_engine = load_paddle_ocr_model()
158 model, processor = load_layoutlmv3_model_and_processor()
159 pii_data, non_pii_raw_text = run_inference_logic(image, model, processor, ocr_engine)
160
161 st.success("Processing Complete!")
162
163 with st.spinner("Generating summary for Non-PII data..."):
164 non_pii_llm_output = generate_non_pii_summary_with_llm(non_pii_raw_text)
165
166 col_pii, col_non_pii = st.columns(2)
167
168 with col_pii:
169 st.header("🔐 PII Data")
170 if pii_data:
171 sorted_pii_labels = sorted(pii_data.keys())
172 for label in sorted_pii_labels:
173 st.subheader(f"🏷️ {label.replace('-', ' ').title()}")
174 for text in pii_data[label]:
175 st.markdown(f"- **{text}**")
176 else:
177 st.info("No PII entities were detected in the document.")
178
179 with col_non_pii:
180 st.header("📝 Non-PII Data")
181 st.markdown(non_pii_llm_output)
182
183# Optional CSS
184st.markdown("""
185<style>
186 .stApp {
187 background-color: #f0f2f6;
188 color: #333333;
189 }
190 .stButton>button {
191 background-color: #4CAF50;
192 color: white;
193 border-radius: 8px;
194 padding: 10px 20px;
195 font-weight: bold;
196 box-shadow: 0 4px 6px rgba(0,0,0,0.1);
197 }
198 .stButton>button:hover {
199 background-color: #45a049;
200 }
201 .stFileUploader {
202 border: 2px dashed #a0aec0;
203 border-radius: 10px;
204 padding: 20px;
205 background-color: #ffffff;
206 }
207 h1 {
208 color: #1a73e8;
209 text-align: center;
210 font-size: 2.5em;
211 }
212 h2 {
213 color: #3f51b5;
214 }
215 h3 {
216 color: #5c6bc0;
217 }
218 ul {
219 list-style-type: none;
220 padding-left: 0;
221 }
222 ul li {
223 margin-bottom: 5px;
224 padding-left: 20px;
225 position: relative;
226 }
227 ul li::before {
228 content: '•';
229 color: #4CAF50;
230 font-weight: bold;
231 display: inline-block;
232 width: 1em;
233 margin-left: -1em;
234 }
235</style>
236""", unsafe_allow_html=True)
237
238# Preprocess OCR results and run inference as described in the full README| Metric | Value |
|---|---|
| Accuracy | ~99.31% |
1@misc{parthesh2025layoutlmv3,
2 title = {LayoutLMv3 Fine-Tuned on Lab Reports with BIOES Tags},
3 author = {Parthesh Ingale},
4 year = {2025},
5 howpublished = {\url{https://huggingface.co/parthesh111/layoutlmv3-finetune-bioes-new}},
6}