AMRPH/

F5-TTS-mansi

lat_to_cyr = {
    "a": "а", "A": "А",
    "e": "е", "E": "Е",
    "o": "о", "O": "О",
    "y": "у", "Y": "У",
}

punct_map = {
        '“': '"', '”': '"', '„': '"', '«': '"', '»': '"',
        '‘': "'", '’': "'", '‚': "'",
        '—': '-', '–': '-', '−': '-',
    }

def normalize_text(text: str) -> str:
    text = unicodedata.normalize("NFD", str(text))
    
    for lat, cyr in lat_to_cyr.items():
        text = text.replace(lat, cyr)
    
    text = unicodedata.normalize("NFC", text)

    text = ''.join(ch for ch in text if unicodedata.category(ch)[0] != 'C')

    text = re.sub(r'\s+', ' ', text)
    
    text = text.strip()
    
    for k, v in punct_map.items():
        text = text.replace(k, v)
    
    return text
    
df['text'] = df['text'].apply(normalize_text)