./llama-cli \
  -hf dahara1/gemma-4-12B-it-qat-UD-japanese-imatrix:gemma-4-12B-it-qat-ja-UD-Q4_K_XL \
  --temp 1.0 \
  --top-p 0.95 \
  --top-k 64 \
  --min-p 0.0 \
  --ctx-size 32000 \
  -hfd dahara1/gemma-4-12B-it-qat-assistant:gemma-4-423M-12b-it-qat-unquantized-assistant-Q4_0 \
  --spec-type draft-mtp \
  --reasoning on

.\llama-server ^
  -hf dahara1/gemma-4-12B-it-qat-UD-japanese-imatrix:gemma-4-12B-it-qat-ja-UD-Q3_K_XL ^
  --host 0.0.0.0 ^
  --port 8080 ^
  --temp 1.0 ^
  --top-p 0.95 ^
  --top-k 64 ^
  --min-p 0.0 ^
  --ctx-size 4096 ^
  --no-mmproj ^
  -hfd dahara1/gemma-4-12B-it-qat-assistant_gguf:gemma-4-423M-12b-it-qat-unquantized-assistant-Q4_0 ^
  --spec-type draft-mtp ^
  --reasoning on ^
  -ub 1024 ^
  -b 1024

./build/bin/llama-server \
  -hf dahara1/gemma-4-12B-it-qat-UD-japanese-imatrix \
  -hff gemma-4-12B-it-qat-ja-UD-Q4_K_XL.gguf \
  --alias gemma-4-12b-it-ja \
  --host 127.0.0.1 \
  --port 8080 \
  --ctx-size 32768 \
  --temp 1.0 \
  --top-p 0.95 \
  --top-k 64 \
  --min-p 0.0 \
  --reasoning off \
  --no-mmproj

#!/usr/bin/env python3
import os
import sys
from openai import OpenAI

BASE_URL = os.getenv("OPENAI_BASE_URL", "http://127.0.0.1:8080/v1")
API_KEY = os.getenv("OPENAI_API_KEY", "sk-no-key-required")
MODEL = os.getenv("OPENAI_MODEL", "gemma-4-12b-it-ja")

prompt = " ".join(sys.argv[1:]) or "魔法少女まどかマギカで一番可愛いのは誰ですか？"

client = OpenAI(
    base_url=BASE_URL,
    api_key=API_KEY,
)

completion = client.chat.completions.create(
    model=MODEL,
    messages=[
        {
            "role": "system",
            "content": "あなたは日本語で簡潔かつ正確に回答するアシスタントです。",
        },
        {
            "role": "user",
            "content": prompt,
        },
    ],
    temperature=1.0,
    top_p=0.95,
    max_tokens=1024,

    # llama-server specific options can be passed via extra_body.
    # If you want to explicitly disable thinking in supported templates,
    # set enable_thinking to False.
    extra_body={
        "top_k": 64,
        "min_p": 0.0,
        "chat_template_kwargs": {
            "enable_thinking": False
        },
    },
)

print(completion.choices[0].message.content)

.//build/bin/llama-server \
  -hf dahara1/gemma-4-12B-it-qat-UD-japanese-imatrix \
  -hff gemma-4-12B-it-qat-ja-UD-Q4_K_XL.gguf \
  --alias gemma-4-12b-it-ja \
  --host 127.0.0.1 \
  --port 8080 \
  --ctx-size 32768 \
  --temp 1.0 \
  --top-p 0.95 \
  --top-k 64 \
  --min-p 0.0 \
  --reasoning on \
  --no-mmproj \
  -hfd dahara1/gemma-4-12B-it-qat-assistant_gguf:gemma-4-423M-12b-it-qat-unquantized-assistant-Q4_0 \
  --spec-type draft-mtp

#!/usr/bin/env python3
import os
import sys
import time
from openai import OpenAI

BASE_URL = os.getenv("OPENAI_BASE_URL", "http://127.0.0.1:8080/v1")
API_KEY = os.getenv("OPENAI_API_KEY", "sk-no-key-required")
MODEL = os.getenv("OPENAI_MODEL", "gemma-4-12b-it-ja")

# 0.0 ならサーバーから来た速度そのまま。
# 0.005 などにするとタイプライター風になる。
TYPE_DELAY = float(os.getenv("TYPE_DELAY", "0.0"))

RESET = "\033[0m"
THINK_COLOR = "\033[90m"      # dark gray
ANSWER_COLOR = "\033[97m"     # bright white
LABEL_COLOR = "\033[36m"      # cyan


def get_delta_field(delta, name: str):
    value = getattr(delta, name, None)
    if value is not None:
        return value

    model_extra = getattr(delta, "model_extra", None)
    if isinstance(model_extra, dict) and name in model_extra:
        return model_extra[name]

    try:
        dumped = delta.model_dump()
        return dumped.get(name)
    except Exception:
        return None


def print_chars(text: str, color: str):
    if not text:
        return

    print(color, end="", flush=True)
    for ch in text:
        print(ch, end="", flush=True)
        if TYPE_DELAY > 0:
            time.sleep(TYPE_DELAY)
    print(RESET, end="", flush=True)


def main():
    prompt = " ".join(sys.argv[1:]) or "世の中の不条理をぶっ壊せ！"

    client = OpenAI(
        base_url=BASE_URL,
        api_key=API_KEY,
    )

    stream = client.chat.completions.create(
        model=MODEL,
        messages=[
            {
                "role": "system",
                "content": "あなたは日本語で簡潔に回答するアシスタントです。",
            },
            {
                "role": "user",
                "content": prompt,
            },
        ],
        stream=True,
        temperature=1.0,
        top_p=0.95,
        max_tokens=2048,
        extra_body={
            "top_k": 64,
            "min_p": 0.0,

            # Gemma 4 の apply_chat_template(..., enable_thinking=True) 相当。
            # llama-server が Jinja chat template にこの値を渡す。
            "chat_template_kwargs": {
                "enable_thinking": True
            },
        },
    )

    phase = None
    thinking_parts = []
    answer_parts = []

    for chunk in stream:
        if not chunk.choices:
            continue

        delta = chunk.choices[0].delta

        reasoning = get_delta_field(delta, "reasoning_content")
        content = get_delta_field(delta, "content")

        if reasoning:
            if phase != "thinking":
                phase = "thinking"
                print_chars("\n[thinking]\n", LABEL_COLOR)
            thinking_parts.append(reasoning)
            print_chars(reasoning, THINK_COLOR)

        if content:
            if phase != "answer":
                phase = "answer"
                print_chars("\n\n[answer]\n", LABEL_COLOR)
            answer_parts.append(content)
            print_chars(content, ANSWER_COLOR)

    print("\n", flush=True)


if __name__ == "__main__":
    main()

../llama.cpp/build/bin/llama-server \
  -hf dahara1/gemma-4-12B-it-qat-UD-japanese-imatrix \
  -hff gemma-4-12B-it-qat-ja-UD-Q4_K_XL.gguf \
  --alias gemma-4-12b-it-ja \
  --host 127.0.0.1 \
  --port 8080 \
  --ctx-size 32768 \
  --temp 1.0 \
  --top-p 0.95 \
  --top-k 64 \
  --min-p 0.0 \
  --reasoning off \
  --image-min-tokens 560 \
  --image-max-tokens 560 \
  -ub 1024

#!/usr/bin/env python3
import base64
import mimetypes
import sys
from pathlib import Path

from openai import OpenAI


BASE_URL = "http://127.0.0.1:8080/v1"
API_KEY = "sk-no-key-required"
MODEL = "gemma-4-12b-it-ja"


def image_to_data_url(path: str) -> str:
    image_path = Path(path)
    if not image_path.exists():
        raise FileNotFoundError(f"Image file not found: {image_path}")

    mime_type, _ = mimetypes.guess_type(image_path)
    if mime_type is None:
        mime_type = "image/png"

    encoded = base64.b64encode(image_path.read_bytes()).decode("utf-8")
    return f"data:{mime_type};base64,{encoded}"


def main() -> None:
    if len(sys.argv) < 2:
        print("Usage: python image_chat_llama_server.py <image_path> [prompt]")
        sys.exit(1)

    image_path = sys.argv[1]
    prompt = " ".join(sys.argv[2:]) or "この画像の内容を日本語で詳しく説明してください。"

    client = OpenAI(
        base_url=BASE_URL,
        api_key=API_KEY,
    )

    response = client.chat.completions.create(
        model=MODEL,
        messages=[
            {
                "role": "user",
                "content": [
                    # Gemma 4 works best when image content comes before text.
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_to_data_url(image_path),
                        },
                    },
                    {
                        "type": "text",
                        "text": prompt,
                    },
                ],
            }
        ],
        temperature=1.0,
        top_p=0.95,
        max_tokens=1024,
        extra_body={
            "top_k": 64,
            "min_p": 0.0,
            "chat_template_kwargs": {
                "enable_thinking": False
            },
        },
    )

    print(response.choices[0].message.content)


if __name__ == "__main__":
    main()

Transcribe the following speech segment in {LANGUAGE} into {LANGUAGE} text.

Follow these specific instructions for formatting the answer:
*   Only output the transcription, with no newlines.
*   When transcribing numbers, write the digits, i.e. write 1.7 and not one point seven, and write 3 instead of three.

Transcribe the following speech segment in {SOURCE_LANGUAGE}, then translate it into {TARGET_LANGUAGE}.
When formatting the answer, first output the transcription in {SOURCE_LANGUAGE}, then one newline, then output the string '{TARGET_LANGUAGE}: ', then the translation in {TARGET_LANGUAGE}.

#!/usr/bin/env python3
import base64
import sys
from pathlib import Path

from openai import OpenAI


BASE_URL = "http://127.0.0.1:8080/v1"
API_KEY = "sk-no-key-required"
MODEL = "gemma-4-12b-it-ja"


DEFAULT_ASR_PROMPT = """Transcribe the following speech segment in Japanese into Japanese text.

Follow these specific instructions for formatting the answer:
* Only output the transcription, with no newlines.
* When transcribing numbers, write the digits, i.e. write 1.7 and not one point seven, and write 3 instead of three."""


def audio_to_base64(path: str) -> tuple[str, str]:
    audio_path = Path(path)
    if not audio_path.exists():
        raise FileNotFoundError(f"Audio file not found: {audio_path}")

    suffix = audio_path.suffix.lower().lstrip(".")
    if suffix not in {"wav", "mp3"}:
        raise ValueError("Please use a .wav or .mp3 file.")

    encoded = base64.b64encode(audio_path.read_bytes()).decode("utf-8")
    return encoded, suffix


def main() -> None:
    if len(sys.argv) < 2:
        print("Usage: python audio_chat_llama_server.py <audio_path> [prompt]")
        sys.exit(1)

    audio_path = sys.argv[1]
    prompt = " ".join(sys.argv[2:]) or DEFAULT_ASR_PROMPT

    encoded_audio, audio_format = audio_to_base64(audio_path)

    client = OpenAI(
        base_url=BASE_URL,
        api_key=API_KEY,
    )

    response = client.chat.completions.create(
        model=MODEL,
        messages=[
            {
                "role": "user",
                "content": [
                    # Gemma 4 works best when audio content comes after text.
                    {
                        "type": "text",
                        "text": prompt,
                    },
                    {
                        "type": "input_audio",
                        "input_audio": {
                            "data": encoded_audio,
                            "format": audio_format,
                        },
                    },
                ],
            }
        ],
        temperature=1.0,
        top_p=0.95,
        max_tokens=2048,
        extra_body={
            "top_k": 64,
            "min_p": 0.0,
            "chat_template_kwargs": {
                "enable_thinking": False
            },
        },
    )

    print(response.choices[0].message.content)


if __name__ == "__main__":
    main()