NeuralNet-Hub/

openchat-3.6-8b-20240522-GGUF

# obtain the official LLaMA model weights and place them in ./models
ls ./models
llama-2-7b tokenizer_checklist.chk tokenizer.model
# [Optional] for models using BPE tokenizers
ls ./models
<folder containing weights and tokenizer json> vocab.json
# [Optional] for PyTorch .bin models like Mistral-7B
ls ./models
<folder containing weights and tokenizer json>

# install Python dependencies
python3 -m pip install -r requirements.txt

# convert the model to ggml FP16 format
python3 convert-hf-to-gguf.py models/mymodel/

# quantize the model to 4-bits (using Q4_K_M method)
./llama-quantize ./models/mymodel/ggml-model-f16.gguf ./models/mymodel/ggml-model-Q4_K_M.gguf Q4_K_M

# update the gguf filetype to current version if older version is now unsupported
./llama-quantize ./models/mymodel/ggml-model-Q4_K_M.gguf ./models/mymodel/ggml-model-Q4_K_M-v2.gguf COPY

<|begin_of_text|><|start_header_id|>System<|end_header_id|>

{system}<|eot_id|><|start_header_id|>GPT4 Correct User<|end_header_id|>

{user}<|eot_id|><|start_header_id|>GPT4 Correct Assistant<|end_header_id|>

{{ if .System }}<|begin_of_text|><|start_header_id|>System<|end_header_id|>

{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>GPT4 Correct User<|end_header_id|>

{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>GPT4 Correct Assistant<|end_header_id|>

{{ .Response }}<|eot_id|>

FROM NeuralNet/openchat-3.6

# sets the temperature to 1 [higher is more creative, lower is more coherent]
PARAMETER temperature 0.5

# sets the context window size to 8192, this controls how many tokens the LLM can use as context to generate the next token
PARAMETER num_ctx 8192

# tokens to generate set to 4096 (max)
PARAMETER num_predict 4096

# set system
SYSTEM "You are an AI assistant created by NeuralNet, your answer are clear and consice"

# template OpenChat3.6
TEMPLATE "{{ if .System }}<|begin_of_text|><|start_header_id|>System<|end_header_id|>

{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>GPT4 Correct User<|end_header_id|>

{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>GPT4 Correct Assistant<|end_header_id|>

{{ .Response }}<|eot_id|>"