1from transformers import AutoTokenizer, AutoModelForSequenceClassification
2import torch
3
4# 加载模型和分词器
5model_name = "GoldWings/vocational-edu-classifier"
6tokenizer = AutoTokenizer.from_pretrained(model_name)
7model = AutoModelForSequenceClassification.from_pretrained(model_name)
8
9# 分类标签
10labels = [
11 "装备制造类",
12 "电子信息类",
13 "财经商贸类",
14 "文化艺术类",
15 "教育体育类"
16]
17
18# 推理
19text = "工业机器人技术、机电一体化技术、数控技术"
20inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=512)
21outputs = model(**inputs)
22predicted_class = torch.argmax(outputs.logits, dim=1).item()
23
24print(f"专业分类结果: {labels[predicted_class]}")
25print(f"置信度: {torch.softmax(outputs.logits, dim=1)[0][predicted_class]:.2%}")
1from transformers import pipeline
2
3classifier = pipeline(
4 "text-classification",
5 model="your-username/vocational-edu-classifier",
6 tokenizer="your-username/vocational-edu-classifier"
7)
8
9# 批量分类
10texts = [
11 "数控技术、模具设计、机械制造",
12 "电子商务、市场营销、现代物流",
13 "计算机网络技术、软件开发、人工智能"
14]
15
16results = classifier(texts)
17for text, result in zip(texts, results):
18 print(f"文本: {text}")
19 print(f"分类: {result['label']} (置信度: {result['score']:.2%})\n")
1import pandas as pd
2from transformers import AutoTokenizer, AutoModelForSequenceClassification
3
4# 加载模型
5model_name = "your-username/vocational-edu-classifier"
6tokenizer = AutoTokenizer.from_pretrained(model_name)
7model = AutoModelForSequenceClassification.from_pretrained(model_name)
8
9labels = ["装备制造", "电子信息", "财经商贸", "文化艺术", "教育体育", "其他"]
10
11def predict_category(text, batch_size=32):
12 """批量预测专业分类"""
13 predictions = []
14
15 # 分批处理
16 for i in range(0, len(texts), batch_size):
17 batch_texts = texts[i:i+batch_size]
18 inputs = tokenizer(batch_texts, return_tensors="pt",
19 truncation=True, max_length=512, padding=True)
20 outputs = model(**inputs)
21 batch_preds = torch.argmax(outputs.logits, dim=1).tolist()
22 predictions.extend([labels[p] for p in batch_preds])
23
24 return predictions
25
26# 读取Excel数据
27df = pd.read_excel("专业数据.xlsx")
28df["预测分类"] = predict_category(df["专业名称"].tolist())
29df.to_excel("分类结果.xlsx", index=False)
30print("分类完成!")
├── config.json # 模型配置文件
├── model.safetensors # 模型权重 (推荐格式)
├── model.bin # 模型权重 (兼容格式)
├── tokenizer.json # 分词器配置
├── tokenizer_config.json # 分词器配置
├── vocab.txt # 词表
├── special_tokens_map.json # 特殊token映射
└── README.md # 本文档
1training_config = {
2 "learning_rate": 2e-5,
3 "per_device_train_batch_size": 32,
4 "per_device_eval_batch_size": 32,
5 "num_train_epochs": 3,
6 "warmup_ratio": 0.1,
7 "weight_decay": 0.01,
8 "logging_steps": 100,
9 "save_steps": 1000,
10 "eval_steps": 1000,
11 "load_best_model_at_end": True,
12 "metric_for_best_model": "f1"
13}
1from transformers import Trainer, TrainingArguments
2from datasets import load_dataset
3
4# 加载数据集
5dataset = load_dataset("csv", data_files="train.csv")
6
7# 预处理
8def tokenize_function(examples):
9 return tokenizer(examples["text"], truncation=True, padding="max_length")
10
11tokenized_dataset = dataset.map(tokenize_function, batched=True)
12
13# 训练参数
14training_args = TrainingArguments(
15 output_dir="./results",
16 num_train_epochs=3,
17 per_device_train_batch_size=32,
18 learning_rate=2e-5,
19 evaluation_strategy="epoch",
20 save_strategy="epoch",
21 load_best_model_at_end=True,
22 metric_for_best_model="f1"
23)
24
25# 开始训练
26trainer = Trainer(
27 model=model,
28 args=training_args,
29 train_dataset=tokenized_dataset["train"],
30 eval_dataset=tokenized_dataset["validation"]
31)
32
33trainer.train()
34trainer.save_model("./fine-tuned-model")
1# 分析专业群建设情况
2major_groups = [
3 "工业机器人技术、机电一体化技术、智能制造装备技术",
4 "电子商务、网络营销、现代物流管理",
5 "大数据技术、人工智能技术应用、计算机应用技术"
6]
7
8results = classifier(major_groups)
9for group, result in zip(major_groups, results):
10 print(f"专业群: {group}")
11 print(f"分类: {result['label']} ({result['score']:.2%})\n")
1# 分析产业人才需求
2industry_text = """
3装备制造业:数控技术人才需求旺盛,月薪 8000-15000 元
4电子信息产业:嵌入式开发、物联网技术人才紧缺
5现代服务业:跨境电商、新媒体运营需求快速增长
6"""
7
8inputs = tokenizer(industry_text, return_tensors="pt")
9outputs = model(**inputs)
10top_predictions = torch.topk(outputs.logits, k=3, dim=1)
11
12print("产业需求分析:")
13for i, (idx, score) in enumerate(zip(top_predictions.indices[0], top_predictions.values[0])):
14 print(f"{i+1}. {labels[idx.item()]}: {torch.softmax(top_predictions.values[0], dim=0)[i]:.2%}")
1@misc{vocational-edu-classifier,
2 author = {Your Name},
3 title = {职业教育专业分类模型 - Vocational Education Text Classifier},
4 year = {2024},
5 publisher = {HuggingFace},
6 url = {https://huggingface.co/your-username/vocational-edu-classifier}
7}