lamm-mit/

BioinspiredMixtral

from llama_cpp import Llama

model_path='lamm-mit/BioinspiredMixtral/ggml-model-q5_K_M.gguf'
chat_format="mistral-instruct"

llm = Llama(model_path=model_path,
            n_gpu_layers=-1,verbose= True, 
            n_ctx=10000,
            #main_gpu=0,
            chat_format=chat_format,
            #split_mode=llama_cpp.LLAMA_SPLIT_LAYER
            )

from llama_cpp import Llama

model_path='lamm-mit/BioinspiredMixtral/ggml-model-q5_K_M.gguf'
chat_format="mistral-instruct"

llm = Llama.from_pretrained(
    repo_id=model_path,
    filename="*q5_K_M.gguf",
    verbose=True,
    n_gpu_layers=-1, 
    n_ctx=10000,
    #main_gpu=0,
    chat_format=chat_format,
)

def generate_BioMixtral (system_prompt='You are an expert in biological materials, mechanics and related topics.', prompt="What is spider silk?",
             temperature=0.0,
             max_tokens=10000,  
             ):

    if system_prompt==None:
        messages=[
            {"role": "user", "content": prompt},
            ]
    else:
        messages=[
            {"role": "system",  "content": system_prompt},
            {"role": "user", "content": prompt},
        ]

    result=llm.create_chat_completion(
            messages=messages,
            temperature=temperature,
            max_tokens=max_tokens,
        )

start_time = time.time()
result=generate_BioMixtral(system_prompt='You respond accurately.', 
                        prompt="What is graphene? Answer with detail.",
                        max_tokens=512, temperature=0.7,  )
print (result)
deltat=time.time() - start_time
print("--- %s seconds ---" % deltat)
toked=tokenizer(res)
print ("Tokens per second (generation): ", len (toked['input_ids'])/deltat)