This model is
google/functiongemma-270m-it converted to
GGUF format for use with llama.cpp, Ollama, LM Studio, and other compatible inference engines.
1from llama_cpp import Llama
2
3# Load the model
4llm = Llama.from_pretrained(
5 repo_id="QuantLLM/functiongemma-270m-it-4bit-gguf",
6 filename="functiongemma-270m-it-4bit-gguf.Q4_K_M.gguf",
7)
8
9# Generate text
10output = llm(
11 "Write a short story about a robot learning to paint:",
12 max_tokens=256,
13 echo=True
14)
15print(output["choices"][0]["text"])
1# Download the model
2huggingface-cli download QuantLLM/functiongemma-270m-it-4bit-gguf functiongemma-270m-it-4bit-gguf.Q4_K_M.gguf --local-dir .
3
4# Create Modelfile
5echo 'FROM ./functiongemma-270m-it-4bit-gguf.Q4_K_M.gguf' > Modelfile
6
7# Import to Ollama
8ollama create functiongemma-270m-it-4bit-gguf -f Modelfile
9
10# Chat with the model
11ollama run functiongemma-270m-it-4bit-gguf
1# Download
2huggingface-cli download QuantLLM/functiongemma-270m-it-4bit-gguf functiongemma-270m-it-4bit-gguf.Q4_K_M.gguf --local-dir .
3
4# Run inference
5./llama-cli -m functiongemma-270m-it-4bit-gguf.Q4_K_M.gguf -p "Hello! " -n 128
