PLTNUM is a protein language model trained to predict protein half-lives based on their sequences.
This model was created based on
westlake-repl/SaProt_650M_AF2 and trained on protein half-life dataset of HeLa human cell line (
paper link).
Use the code below to get started with the model.
1from torch import sigmoid
2import torch.nn as nn
3from transformers import AutoModel, AutoConfig, PreTrainedModel, AutoTokenizer
4
5
6class PLTNUM_PreTrainedModel(PreTrainedModel):
7 config_class = AutoConfig
8
9 def __init__(self, config):
10 super(PLTNUM_PreTrainedModel, self).__init__(config)
11 self.model = AutoModel.from_pretrained(self.config._name_or_path)
12
13 self.fc_dropout1 = nn.Dropout(0.8)
14 self.fc_dropout2 = nn.Dropout(0.4)
15 self.fc = nn.Linear(self.config.hidden_size, 1)
16 self._init_weights(self.fc)
17
18 def _init_weights(self, module):
19 if isinstance(module, nn.Linear):
20 nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
21 if module.bias is not None:
22 nn.init.constant_(module.bias, 0)
23 elif isinstance(module, nn.Embedding):
24 nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
25 if module.padding_idx is not None:
26 nn.init.constant_(module.weight[module.padding_idx], 0.0)
27 elif isinstance(module, nn.LayerNorm):
28 nn.init.constant_(module.bias, 0)
29 nn.init.constant_(module.weight, 1.0)
30
31 def forward(self, inputs):
32 outputs = self.model(**inputs)
33 last_hidden_state = outputs.last_hidden_state[:, 0]
34 output = (
35 self.fc(self.fc_dropout1(last_hidden_state))
36 + self.fc(self.fc_dropout2(last_hidden_state))
37 ) / 2
38 return output
39
40 def create_embedding(self, inputs):
41 outputs = self.model(**inputs)
42 last_hidden_state = outputs.last_hidden_state[:, 0]
43 return last_hidden_state
44
45
46model = PLTNUM_PreTrainedModel.from_pretrained("sagawa/PLTNUM-SaProt-HeLa")
47tokenizer = AutoTokenizer.from_pretrained("sagawa/PLTNUM-SaProt-HeLa")
48seq = "MdSdGdRdGdKpQpGpGpKdApRpApKpAdKdTaRpScSvRvAlGvLaQpFfPrVlGvRvVqHvRvLvLvRvKvGvNpYpSdEpRdVdGdAsGcAnPsVsYvLvArAvVvLvErYvLvTvAvEqIlLcEvLqAlGcNvAqAcRvDvNvKvKhTrRdIrIdPlRlHsLsQqLvAsIqRcNvDdEpEvLsNcKvLvLcGvRpVpTdIrApQpGnGdVhLdPdNdIdQdApVvLpLdPdKdKdTdEpSpHpHpKpPpKpGdKd"
49input = tokenizer(
50 [seq],
51 add_special_tokens=True,
52 max_length=512,
53 padding="max_length",
54 truncation=True,
55 return_offsets_mapping=False,
56 return_attention_mask=True,
57 return_tensors="pt",
58)
59print(sigmoid(model(input)))
Prediction of Protein Half-lives from Amino Acid Sequences by Protein Language Models
Tatsuya Sagawa, Eisuke Kanao, Kosuke Ogata, Koshi Imami, Yasushi Ishihama
bioRxiv 2024.09.10.612367; doi:
https://doi.org/10.1101/2024.09.10.612367
