Built on
ai4bharat/IndicBERTv2-MLM-only — a 278M parameter BERT model pretrained on 20.9B tokens across 24 Indian languages.
1from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline
2
3model = AutoModelForTokenClassification.from_pretrained("hiteshwadhwani/pii-model-indicv2")
4tokenizer = AutoTokenizer.from_pretrained("hiteshwadhwani/pii-model-indicv2")
5
6ner = pipeline("ner", model=model, tokenizer=tokenizer, aggregation_strategy="first")
7
8# English
9results = ner("Mr John Smith lives at 456 Oak Avenue Boston")
10
11# Hinglish
12results = ner("Mera naam Rajesh Kumar hai aur main Mumbai mein rehta hoon")
13
14# Hindi (Devanagari)
15results = ner("मेरा नाम राजेश कुमार है और मैं मुंबई में रहता हूं")
16
17# Hindi amounts and dates in words
18results = ner("आपके क्रेडिट कार्ड के बीस हज़ार रूपये शुल्क लंबित हैं, जो पंद्रह मार्च को देना था।")
19
20for entity in results:
21 if entity["entity_group"] != "O":
22 print(f"{entity['word']} → {entity['entity_group']} ({entity['score']:.2f})")
1def redact_pii(text, ner_pipeline, threshold=0.5):
2 results = ner_pipeline(text)
3 entities = [r for r in results if r["score"] >= threshold and r["entity_group"] != "O"]
4 entities.sort(key=lambda x: x["start"])
5
6 merged = []
7 for ent in entities:
8 label = ent["entity_group"]
9 if merged and merged[-1]["label"] == label and ent["start"] <= merged[-1]["end"] + 1:
10 merged[-1]["end"] = max(merged[-1]["end"], ent["end"])
11 else:
12 merged.append({"label": label, "start": ent["start"], "end": ent["end"]})
13
14 redacted = text
15 for span in reversed(merged):
16 redacted = redacted[:span["start"]] + f"[{span['label']}]" + redacted[span["end"]:]
17 return redacted
18
19print(redact_pii("Shri Rajesh Kumar lives at 42 MG Road Bengaluru Karnataka", ner))
20# [PREFIX] [FIRSTNAME] [LASTNAME] lives at [BUILDINGNUMBER] [STREET] [CITY] [STATE]
21
22print(redact_pii("आपके बीस हज़ार रूपये शुल्क लंबित हैं, जो पंद्रह मार्च को देना था।", ner))
23# आपके [AMOUNT] शुल्क लंबित हैं, जो [DATE] को देना था।
