Views
No views yet
import spacy
def clean_up_sentence(text: str) -> str:
text = text.replace("---", "")
text = text.replace("\n", " ")
text = text.strip()
if not text.endswith(('.', '!', '?', ":")):
# Since we are breaking a longer text into sentences ourselves, we should always end a sentence with a period.
text = text + "."
return text
sentence_splitter = spacy.load("en_core_web_sm")
spacy_document = sentence_splitter("This is a long text. It has two or more sentence. Spacy will break it down into sentences.")
results = []
for sentence in spacy_document.sents:
clean_text = clean_up_sentence(str(sentence))
classification = grammar_checker(clean_text)[0]
results.append({
"label": classification['label'],
"score": classification['score'],
"sentence": clean_text
})
pd.DataFrame.from_dict(results)| Training Loss | Epoch | Step | Validation Loss | Accuracy | Matthews Correlation |
|---|---|---|---|---|---|
| 0.363 | 1.0 | 200000 | 0.3634 | 0.8487 | 0.7025 |
| 0.3032 | 2.0 | 400000 | 0.3546 | 0.8577 | 0.7192 |