Views
No views yet
neuronx compatible checkpoint for meta-llama/Llama-2-7b-chat-hf. You can find detailed information about the base model on its Model Card.1
2from optimum.neuron import pipeline
3
4# Load pipeline from Hugging Face repository
5pipe = pipeline("text-generation", "WaelDataReply/Llama-2-7b-chat-seqlen-4096-bs-2")
6
7# We use the tokenizer's chat template to format each message - see https://huggingface.co/docs/transformers/main/en/chat_templating
8messages = [
9 {"role": "user", "content": "What is the difference between AWS Step Functions and Apache Airflow?"},
10]
11prompt = pipe.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
12# Run generation
13outputs = pipe(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
14print(outputs[0]["generated_text"])
151{
2 "num_cores": 2,
3 "auto_cast_type": "fp16"
4}1{
2 "sequence_length": 4096,
3 "batch_size": 2
4}