这是一个从零开始训练的GPT-2架构语言模型，专门用于**因果语言建模（Causal Language Modeling）**任务。模型能够根据给定的前文序列预测下一个token，实现文本续写和生成功能。
1from transformers import GPT2LMHeadModel, GPT2Tokenizer
2
3# 加载模型和tokenizer
4model = GPT2LMHeadModel.from_pretrained("ludandaye/gpt-causal-lm")
5tokenizer = GPT2Tokenizer.from_pretrained("ludandaye/gpt-causal-lm")
6
7# 设置为评估模式
8model.eval()
1import torch
2
3# 准备输入
4prompt = "1 2 3"
5inputs = tokenizer(prompt, return_tensors="pt")
6
7# 生成文本
8with torch.no_grad():
9 outputs = model.generate(
10 inputs.input_ids,
11 max_new_tokens=10,
12 temperature=0.8,
13 do_sample=True,
14 pad_token_id=tokenizer.pad_token_id,
15 eos_token_id=tokenizer.eos_token_id
16 )
17
18# 解码结果
19generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
20print(generated_text)
1import torch
2
3# 准备输入
4prompt = "1 2 3"
5inputs = tokenizer(prompt, return_tensors="pt")
6
7# 预测下一个token
8with torch.no_grad():
9 outputs = model(inputs.input_ids)
10 logits = outputs.logits
11 next_token_logits = logits[0, -1, :]
12 probs = torch.softmax(next_token_logits, dim=-1)
13
14# 获取top-k tokens
15top_k = 5
16top_probs, top_indices = torch.topk(probs, top_k)
17
18for prob, idx in zip(top_probs, top_indices):
19 token = tokenizer.decode([idx])
20 print(f"Token: {token}, Probability: {prob.item():.4f}")
1@misc{gpt-causal-lm-2024,
2 author = {ludandaye},
3 title = {GPT Causal Language Model - From Scratch Training},
4 year = {2024},
5 publisher = {Hugging Face},
6 url = {https://huggingface.co/ludandaye/gpt-causal-lm}
7}
