Deepseek Coder is composed of a series of code language models, each trained from scratch on 2T tokens, with a composition of 87% code and 13% natural language in both English and Chinese. We provide various sizes of the code model, ranging from 1B to 33B versions. Each model is pre-trained on project-level code corpus by employing a window size of 16K and a extra fill-in-the-blank task, to support project-level code completion and infilling. For coding capabilities, Deepseek Coder achieves state-of-the-art performance among open-source code models on multiple programming languages and various benchmarks.
-
Massive Training Data: Trained from scratch on 2T tokens, including 87% code and 13% linguistic data in both English and Chinese languages.
-
Highly Flexible & Scalable: Offered in model sizes of 1.3B, 5.7B, 6.7B, and 33B, enabling users to choose the setup most suitable for their requirements.
-
Superior Model Performance: State-of-the-art performance among publicly available code models on HumanEval, MultiPL-E, MBPP, DS-1000, and APPS benchmarks.
-
Advanced Code Completion Capabilities: A window size of 16K and a fill-in-the-blank task, supporting project-level code completion and infilling tasks.
deepseek-coder-1.3b-base is a 1.3B parameter model with Multi-Head Attention trained on 1 trillion tokens.
Here give some examples of how to use our model.
1from transformers import AutoTokenizer, AutoModelForCausalLM
2import torch
3tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/deepseek-coder-1.3b-base", trust_remote_code=True)
4model = AutoModelForCausalLM.from_pretrained("deepseek-ai/deepseek-coder-1.3b-base", trust_remote_code=True).cuda()
5input_text = "#write a quick sort algorithm"
6inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
7outputs = model.generate(**inputs, max_length=128)
8print(tokenizer.decode(outputs[0], skip_special_tokens=True))
1from transformers import AutoTokenizer, AutoModelForCausalLM
2import torch
3tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/deepseek-coder-1.3b-base", trust_remote_code=True)
4model = AutoModelForCausalLM.from_pretrained("deepseek-ai/deepseek-coder-1.3b-base", trust_remote_code=True).cuda()
5input_text = """<｜fim▁begin｜>def quick_sort(arr):
6 if len(arr) <= 1:
7 return arr
8 pivot = arr[0]
9 left = []
10 right = []
11<｜fim▁hole｜>
12 if arr[i] < pivot:
13 left.append(arr[i])
14 else:
15 right.append(arr[i])
16 return quick_sort(left) + [pivot] + quick_sort(right)<｜fim▁end｜>"""
17inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
18outputs = model.generate(**inputs, max_length=128)
19print(tokenizer.decode(outputs[0], skip_special_tokens=True)[len(input_text):])
1from transformers import AutoTokenizer, AutoModelForCausalLM
2tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/deepseek-coder-1.3b-base", trust_remote_code=True)
3model = AutoModelForCausalLM.from_pretrained("deepseek-ai/deepseek-coder-1.3b-base", trust_remote_code=True).cuda()
4
5input_text = """#utils.py
6import torch
7from sklearn import datasets
8from sklearn.model_selection import train_test_split
9from sklearn.preprocessing import StandardScaler
10from sklearn.metrics import accuracy_score
11
12def load_data():
13 iris = datasets.load_iris()
14 X = iris.data
15 y = iris.target
16
17 # Standardize the data
18 scaler = StandardScaler()
19 X = scaler.fit_transform(X)
20
21 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
22
23 # Convert numpy data to PyTorch tensors
24 X_train = torch.tensor(X_train, dtype=torch.float32)
25 X_test = torch.tensor(X_test, dtype=torch.float32)
26 y_train = torch.tensor(y_train, dtype=torch.int64)
27 y_test = torch.tensor(y_test, dtype=torch.int64)
28
29 return X_train, X_test, y_train, y_test
30
31def evaluate_predictions(y_test, y_pred):
32 return accuracy_score(y_test, y_pred)
33#model.py
34import torch
35import torch.nn as nn
36import torch.optim as optim
37from torch.utils.data import DataLoader, TensorDataset
38
39class IrisClassifier(nn.Module):
40 def __init__(self):
41 super(IrisClassifier, self).__init__()
42 self.fc = nn.Sequential(
43 nn.Linear(4, 16),
44 nn.ReLU(),
45 nn.Linear(16, 3)
46 )
47
48 def forward(self, x):
49 return self.fc(x)
50
51 def train_model(self, X_train, y_train, epochs, lr, batch_size):
52 criterion = nn.CrossEntropyLoss()
53 optimizer = optim.Adam(self.parameters(), lr=lr)
54
55 # Create DataLoader for batches
56 dataset = TensorDataset(X_train, y_train)
57 dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)
58
59 for epoch in range(epochs):
60 for batch_X, batch_y in dataloader:
61 optimizer.zero_grad()
62 outputs = self(batch_X)
63 loss = criterion(outputs, batch_y)
64 loss.backward()
65 optimizer.step()
66
67 def predict(self, X_test):
68 with torch.no_grad():
69 outputs = self(X_test)
70 _, predicted = outputs.max(1)
71 return predicted.numpy()
72#main.py
73from utils import load_data, evaluate_predictions
74from model import IrisClassifier as Classifier
75
76def main():
77 # Model training and evaluation
78"""
79inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
80outputs = model.generate(**inputs, max_new_tokens=140)
81print(tokenizer.decode(outputs[0]))
This code repository is licensed under the MIT License. The use of DeepSeek Coder models is subject to the Model License. DeepSeek Coder supports commercial use.
If you have any questions, please raise an issue or contact us at
agi_code@deepseek.com.
