You can deploy this model and start earning money today!
guo1006/
Pegasus-523M-Chinese-finetuned-lcsts-accelerate

Name: Pegasus-523M-Chinese-finetuned-lcsts-accelerate
Availability: InStock
pytorch
pegasus
apache-2.0
Views
No views yet
用于中文文本摘要任务

使用方法（推理与评估）

python
1. 环境准备
!git clone https://github.com/IDEA-CCNL/Fengshenbang-LM
!ls ./Fengshenbang-LM/fengshen/examples/pegasus/
import sys
# 添加 Fengshenbang-LM 到 sys.path
sys.path.append('./Fengshenbang-LM/')

2. 加载模型和分词器
from fengshen.examples.pegasus.tokenizers_pegasus import PegasusTokenizer
from transformers import PegasusForConditionalGeneration

# 初始化分词器
tokenizer = PegasusTokenizer.from_pretrained("guo1006/Pegasus-523M-Chinese-finetuned-lcsts-accelerate")

# 加载微调后的模型
model = PegasusForConditionalGeneration.from_pretrained(
    "guo1006/Pegasus-523M-Chinese-finetuned-lcsts-accelerate"
)
import jieba
import evaluate
from rouge_score import rouge_scorer, scoring
import datasets

3. 中文 ROUGE 评估指标
class ChineseRouge(evaluate.Metric):
    def _info(self):
        return evaluate.MetricInfo(
            description="中文 ROUGE 分数（基于 jieba 分词）",
            citation="https://github.com/google-research/google-research/tree/master/rouge",
            inputs_description="预测和参考均为原始中文字符串列表",
            features=datasets.Features({
                "predictions": datasets.Value("string"),
                "references": datasets.Value("string"),
            }),
            reference_urls=["https://en.wikipedia.org/wiki/ROUGE_(metric)"],
        )

    def _compute(self, predictions, references, rouge_types=None, use_stemmer=False):
        if rouge_types is None:
            rouge_types = ["rouge1", "rouge2", "rougeL"]

        class JiebaTokenizer:
            def tokenize(self, text):
                return jieba.lcut(text)

        tokenizer = JiebaTokenizer()
        scorer = rouge_scorer.RougeScorer(
            rouge_types=rouge_types,
            use_stemmer=use_stemmer,
            tokenizer=tokenizer
        )
        aggregator = scoring.BootstrapAggregator()

        for pred, ref in zip(predictions, references):
            scores = scorer.score(ref, pred)
            aggregator.add_scores(scores)

        result = aggregator.aggregate()
        return {key: result[key].mid.fmeasure for key in result}
from transformers import pipeline
4. 文本摘要示例
# 加载Pipeline
hub_model_id = "guo1006/Pegasus-523M-Chinese-finetuned-lcsts-accelerate"
summarizer = pipeline("summarization", model=hub_model_id, tokenizer=tokenizer, framework='pt')

# 推理示例
sample_text = dataset["val"][0]["text"]
generated_summary = summarizer(sample_text)[0]['summary_text']
print(generated_summary)
5. ROUGE评估示例
rouge = ChineseRouge()
# 添加预测和参考数据
rouge.add_batch(
    predictions=[generated_summary],
    references=[dataset["val"][0]["summary"]]
)
# 计算ROUGE得分
result = rouge.compute()
print(result)