pip install git+https://github.com/lightblue-tech/shitsu.git1from shitsu import ShitsuScorer
2
3text_list = [
4 "Photosynthesis is a system of biological processes by which photosynthetic organisms, such as most plants, algae, and cyanobacteria, convert light energy, typically from sunlight, into the chemical energy necessary to fuel their metabolism.",
5 "Congratulations! You have all been selected to receive a free gift card worth $1000. Click on this link [Link] to claim your reward now. Limited time offer, so act fast! Don't miss out on this amazing opportunity."]
6
7# Choose a language from one of: ['am', 'ar', 'bg', 'bn', 'cs', 'da', 'de', 'el', 'en', 'es', 'fa', 'fi', 'fr', 'gu', 'ha', 'hi', 'hu', 'id', 'it', 'ja', 'jv', 'kn', 'ko', 'lt', 'mr', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sv', 'sw', 'ta', 'te', 'th', 'tl', 'tr', 'uk', 'ur', 'vi', 'yo', 'zh']
8language_code = "en"
9scorer = ShitsuScorer(language_code)
10scores = scorer.score(text_list)
11scores
12# array([ 0.9897383 , -0.08109612], dtype=float32)1
2from safetensors.torch import load_model
3import fasttext
4from huggingface_hub import hf_hub_download
5from tqdm.auto import tqdm
6import torch
7import numpy as np
8import torch
9import torch.nn as nn
10
11class FasttextEmbedRegressor(nn.Module):
12 def __init__(self, input_size=300):
13 super(FasttextEmbedRegressor, self).__init__()
14 layer_1_size = 64
15 layer_2_size = 32
16 self.fc1 = nn.Linear(input_size, layer_1_size)
17 self.fc2 = nn.Linear(layer_1_size, layer_2_size)
18 self.fc3 = nn.Linear(layer_2_size, 1)
19
20 def forward(self, x):
21 x = torch.relu(self.fc1(x))
22 x = torch.relu(self.fc2(x))
23 x = self.fc3(x)
24 return x
25
26class ShitsuScorer:
27 def __init__(self, lang_code):
28 fasttext_model_path = hf_hub_download(repo_id=f"facebook/fasttext-{lang_code}-vectors", filename="model.bin")
29 self.fasttext_model = fasttext.load_model(fasttext_model_path)
30 self.regressor_model = FasttextEmbedRegressor().eval()
31 regressor_model_path = hf_hub_download(repo_id=f"lightblue/shitsu_text_scorer", filename=f"{lang_code}.safetensors")
32 load_model(self.regressor_model, regressor_model_path)
33
34 def score(self, text_list):
35 embeddings = np.stack([self.fasttext_model.get_sentence_vector(x.replace("\n", " ")) for x in tqdm(text_list)])
36 return self.regressor_model(torch.Tensor(embeddings)).detach().numpy().flatten()
37
38text_list = [
39 "Photosynthesis is a system of biological processes by which photosynthetic organisms, such as most plants, algae, and cyanobacteria, convert light energy, typically from sunlight, into the chemical energy necessary to fuel their metabolism.",
40 "Congratulations! You have all been selected to receive a free gift card worth $1000. Click on this link [Link] to claim your reward now. Limited time offer, so act fast! Don't miss out on this amazing opportunity."]
41
42scorer = ShitsuScorer("en")
43scores = scorer.score(text_list)
44scores
45# array([ 0.9897383 , -0.08109612], dtype=float32)1system_message = """You are a text filtering AI model.
2Your input is a piece of text.
3Your output is a score of how likely the text is to appear in a useful {language} textbook, encyclopedia, or any other important document.
4
5Output your score on a scale of 0-100, with 0 meaning that the text contains no useful {language} information and 100 meaning that the text is very useful and is exceedingly likely to appear in a {language} textbook, encyclopedia, or any other important document. If the text is not mostly fluent, natural {language}, output 0.
6
7Your output should be only an integer from 0-100."""