import re
import ftfy
from datasets import load_dataset
from tqdm import tqdm
import pandas as pd
def process_text(text):
# use ftfy
text = ftfy.fix_text(text)
# unify newline style
text = text.replace("\r", "\n")
# replace tabs
text = text.replace("\t", " ")
# replace fancy double quotestext = re.sub(r"[“”]", '"', text)
# replace fancy single quotes
text = re.sub(r"[‘’]", "'", text)
# replace double single quotes with double quotes
text =… See the full description on the dataset page: https://huggingface.co/datasets/PJMixers-Dev/Nelathan_synthetic-sugar-quill-cleaner.
