import re

chars_to_ignore_regex = '[\,\?\.\!\;\:\"\”\’\'\“\(\)\[\\\\&/！\‘]' # delete following chars
chars_to_space_regex = '[\–\—\-]' # replace the following chars into space

def remove_special_characters(batch):
    batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower() + " "
    batch["sentence"] = re.sub(chars_to_space_regex, ' ', batch["sentence"]) + " "
    # replacing some character
    batch["sentence"] = batch["sentence"].replace("é", "e").replace("á", "a").replace("ł", "l").replace("ń", "n").replace("ō", "o").strip()
    return batch

common_voice = common_voice.map(remove_special_characters)