This is a
ctranslate2 v4.5.0 int8 conversion of
meta-llama/Meta-Llama-3-8B-Instruct created with:
ct2 doesn't have hf-hub integration, so you'll need to manually download the model files:
1import sys
2import ctranslate2
3from transformers import AutoTokenizer
4
5model_dir = sys.argv[1] # download dir
6tokenizer_dir = meta-llama/Meta-Llama-3-8B-Instruct
7
8print("Loading the model...")
9generator = ctranslate2.Generator(model_dir, device="cuda")
10tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir)
11
12dialog = [{"role": "user", "content": "What is the meaning of life, the universe and everything?"}]
13max_generation_length = 512
14
15prompt_string = tokenizer.apply_chat_template(dialog, add_generation_prompt=True, tokenize=False)
16# It seems silly to tokenize=False and then call tokenize, but tokenize=True returns just ids; we need actual tokens
17prompt_tokens = tokenizer.tokenize(prompt_string)
18
19step_results = generator.generate_tokens(
20 prompt_tokens,
21 max_length=max_generation_length,
22 sampling_temperature=0.6,
23 sampling_topk=20,
24 sampling_topp=1,
25)
26for step_result in step_results:
27 word = tokenizer.decode([step_result.token_id])
28 print(word, end="", flush=True)
