Views
No views yet
| Model | HumanEval(pass@1) | Date |
|---|---|---|
| CodeFuse-DeepSeek-33B | 78.65% | 2024.01 |
| CodeFuse-Mixtral-8x7B | 56.10% | 2024.01 |
| CodeFuse-CodeLlama-34B | 74.4% | 2023.9 |
| CodeFuse-CodeLlama-34B-4bits | 73.8% | 2023.9 |
| CodeFuse-StarCoder-15B | 54.9% | 2023.9 |
| CodeFuse-QWen-14B | 48.78% | 2023.10 |
| CodeFuse-CodeGeeX2-6B | 45.12% | 2023.11 |
| WizardCoder-Python-34B-V1.0 | 73.2% | 2023.8 |
| GPT-4(zero-shot) | 67.0% | 2023.3 |
| PanGu-Coder2 15B | 61.6% | 2023.8 |
| CodeLlama-34b-Python | 53.7% | 2023.8 |
| CodeLlama-34b | 48.8% | 2023.8 |
| GPT-3.5(zero-shot) | 48.1% | 2022.11 |
| OctoCoder | 46.2% | 2023.8 |
| StarCoder-15B | 33.6% | 2023.5 |
| Qwen-14b | 32.3% | 2023.10 |
1"""
2<s>system
3System instruction
4<s>human
5Human 1st round input
6<s>bot
7Bot 1st round output<｜end▁of▁sentence｜>
8<s>human
9Human 2nd round input
10<s>bot
11Bot 2nd round output<｜end▁of▁sentence｜>
12...
13...
14...
15<s>human
16Human nth round input
17<s>bot
18"""1"""
2<s>human
3User prompt...
4<s>bot
5
6"""<s>human
# language: Python
from typing import List
def separate_paren_groups(paren_string: str) -> List[str]:
""" Input to this function is a string containing multiple groups of nested parentheses. Your goal is to
separate those group into separate strings and return the list of those.
Separate groups are balanced (each open brace is properly closed) and not nested within each other
Ignore any spaces in the input string.
>>> separate_paren_groups('( ) (( )) (( )( ))')
['()', '(())', '(()())']
"""
<s>bot
# language: Python" for Python) used by CodeGeex models.1import torch
2from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
3
4model_dir = "codefuse-ai/CodeFuse-DeepSeek-33B"
5
6def load_model_tokenizer(model_path):
7 tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
8 tokenizer.eos_token = "<｜end▁of▁sentence｜>"
9 tokenizer.pad_token = "<｜end▁of▁sentence｜>"
10 tokenizer.eos_token_id = tokenizer.convert_tokens_to_ids(tokenizer.eos_token)
11 tokenizer.pad_token_id = tokenizer.convert_tokens_to_ids(tokenizer.pad_token)
12 tokenizer.padding_side = "left"
13
14 model = AutoModelForCausalLM.from_pretrained(model_path, device_map='auto',torch_dtype=torch.bfloat16, trust_remote_code=True)
15 return model, tokenizer
16
17
18HUMAN_ROLE_START_TAG = "<s>human\n"
19BOT_ROLE_START_TAG = "<s>bot\n"
20
21text_list = [f'{HUMAN_ROLE_START_TAG}Write a QuickSort program\n#Python\n{BOT_ROLE_START_TAG}']
22
23model, tokenizer = load_model_tokenizer(model_dir)
24inputs = tokenizer(text_list, return_tensors='pt', padding=True, add_special_tokens=False).to('cuda')
25input_ids = inputs["input_ids"]
26attention_mask = inputs["attention_mask"]
27generation_config = GenerationConfig(
28 eos_token_id=tokenizer.eos_token_id,
29 pad_token_id=tokenizer.pad_token_id,
30 temperature=0.1,
31 max_new_tokens=512,
32 num_return_sequences=1,
33 num_beams=1,
34 top_p=0.95,
35 do_sample=False
36)
37outputs = model.generate(
38 inputs= input_ids,
39 attention_mask=attention_mask,
40 **generation_config.to_dict()
41)
42gen_text = tokenizer.batch_decode(outputs[:, input_ids.shape[1]:], skip_special_tokens=True)
43print(gen_text[0])| 模型 | HumanEval(pass@1) | 日期 |
|---|---|---|
| CodeFuse-CodeLlama-34B | 74.4% | 2023.9 |
| CodeFuse-CodeLlama-34B-4bits | 73.8% | 2023.9 |
| WizardCoder-Python-34B-V1.0 | 73.2% | 2023.8 |
| GPT-4(zero-shot) | 67.0% | 2023.3 |
| PanGu-Coder2 15B | 61.6% | 2023.8 |
| CodeLlama-34b-Python | 53.7% | 2023.8 |
| CodeLlama-34b | 48.8% | 2023.8 |
| GPT-3.5(zero-shot) | 48.1% | 2022.11 |
| OctoCoder | 46.2% | 2023.8 |
| StarCoder-15B | 33.6% | 2023.5 |
| Qwen-14b | 32.3% | 2023.10 |
| CodeFuse-StarCoder-15B | 54.9% | 2023.9 |
| CodeFuse-QWen-14B | 48.78% | 2023.8 |
| CodeFuse-CodeGeeX2-6B | 45.12% | 2023.11 |
| CodeFuse-DeepSeek-33B. | 78.65% | 2024.01 |
1"""
2<s>system
3System instruction
4<s>human
5Human 1st round input
6<s>bot
7Bot 1st round output<｜end▁of▁sentence｜>
8<s>human
9Human 2nd round input
10<s>bot
11Bot 2nd round output<｜end▁of▁sentence｜>
12...
13...
14...
15<s>human
16Human nth round input
17<s>bot
18"""1"""
2<s>human
3User prompt...
4<s>bot
5
6"""1<s>human
2# language: Python
3from typing import List
4def separate_paren_groups(paren_string: str) -> List[str]:
5 """ Input to this function is a string containing multiple groups of nested parentheses. Your goal is to
6 separate those group into separate strings and return the list of those.
7 Separate groups are balanced (each open brace is properly closed) and not nested within each other
8 Ignore any spaces in the input string.
9 >>> separate_paren_groups('( ) (( )) (( )( ))')
10 ['()', '(())', '(()())']
11 """
12<s>bot
13# language: Python"）。1import torch
2from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
3
4model_dir = "codefuse-ai/CodeFuse-DeepSeek-33B"
5
6def load_model_tokenizer(model_path):
7 tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
8 tokenizer.eos_token = "<｜end▁of▁sentence｜>"
9 tokenizer.pad_token = "<｜end▁of▁sentence｜>"
10 tokenizer.eos_token_id = tokenizer.convert_tokens_to_ids(tokenizer.eos_token)
11 tokenizer.pad_token_id = tokenizer.convert_tokens_to_ids(tokenizer.pad_token)
12 tokenizer.padding_side = "left"
13
14 model = AutoModelForCausalLM.from_pretrained(model_path, device_map='auto',torch_dtype=torch.bfloat16, trust_remote_code=True)
15 return model, tokenizer
16
17HUMAN_ROLE_START_TAG = "<s>human\n"
18BOT_ROLE_START_TAG = "<s>bot\n"
19
20text_list = [f'{HUMAN_ROLE_START_TAG}请写一个快排程序\n#Python\n{BOT_ROLE_START_TAG}']
21
22model, tokenizer = load_model_tokenizer(model_dir)
23inputs = tokenizer(text_list, return_tensors='pt', padding=True, add_special_tokens=False).to('cuda')
24input_ids = inputs["input_ids"]
25attention_mask = inputs["attention_mask"]
26generation_config = GenerationConfig(
27 eos_token_id=tokenizer.eos_token_id,
28 pad_token_id=tokenizer.pad_token_id,
29 temperature=0.2,
30 max_new_tokens=512,
31 num_return_sequences=1,
32 num_beams=1,
33 top_p=0.95,
34 do_sample=False
35)
36outputs = model.generate(
37 inputs= input_ids,
38 attention_mask=attention_mask,
39 **generation_config.to_dict()
40)
41gen_text = tokenizer.batch_decode(outputs[:, input_ids.shape[1]:], skip_special_tokens=True)
42print(gen_text[0])