Built on top of
ai4bharat/indic-bert — a multilingual ALBERT model pretrained on 12 Indian languages.
1from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline
2
3model = AutoModelForTokenClassification.from_pretrained("hiteshwadhwani/pii-model-indic-v1")
4tokenizer = AutoTokenizer.from_pretrained("hiteshwadhwani/pii-model-indic-v1")
5
6ner = pipeline("ner", model=model, tokenizer=tokenizer, aggregation_strategy="first")
7
8# English
9results = ner("Mr John Smith lives at 456 Oak Avenue Boston")
10
11# Hinglish
12results = ner("Mera naam Rajesh Kumar hai aur main Mumbai mein rehta hoon")
13
14# Hindi (Devanagari)
15results = ner("कृपया प्रिया शर्मा को +91 98765 43210 पर call करें")
16
17# Devanagari names
18results = ner("राजेश कुमार का account number 1234567890 है")
19
20for entity in results:
21 print(f"{entity['word']} → {entity['entity_group']} ({entity['score']:.2f})")
1def redact_pii(text, ner_pipeline, threshold=0.85):
2 results = ner_pipeline(text)
3 entities = [r for r in results if r["score"] >= threshold and r["entity_group"] != "O"]
4 entities.sort(key=lambda x: x["start"])
5
6 merged = []
7 for ent in entities:
8 label = ent["entity_group"]
9 if merged and merged[-1]["label"] == label and ent["start"] <= merged[-1]["end"] + 1:
10 merged[-1]["end"] = max(merged[-1]["end"], ent["end"])
11 else:
12 merged.append({"label": label, "start": ent["start"], "end": ent["end"]})
13
14 redacted = text
15 for span in reversed(merged):
16 redacted = redacted[:span["start"]] + f"[{span['label']}]" + redacted[span["end"]:]
17 return redacted
18
19print(redact_pii("Shri Rajesh Kumar lives at 42 MG Road Bengaluru Karnataka", ner))
20# [PREFIX] [FIRSTNAME] [LASTNAME] lives at [BUILDINGNUMBER] [STREET] [CITY] [STATE]
