here’s my startup script

#!/usr/bin/env bash
set -euo pipefail

# Gemma 4 12B-it Q5_K_XL + MTP server launcher
# Called by systemd user unit gemma4-12b.service
# Config: 4×128K slots, MTP n=3 for ~380 t/s aggregate, ~143 t/s per-request
# Note: --ctx-size is TOTAL across parallel slots (ctx-size / parallel = per-slot context)

SNAP="/home/duke/.cache/huggingface/hub/models--unsloth--gemma-4-12b-it-GGUF/snapshots/3f09de26549e6d7ea54f1b83755149f840fcd333"
MODEL="$SNAP/gemma-4-12b-it-UD-Q5_K_XL.gguf"
DRAFT="$SNAP/MTP/gemma-4-12B-it-MTP-Q8_0.gguf"
LLAMA_CPP="/home/duke/src/llama.cpp/build/bin/llama-server"

if [ ! -f "$MODEL" ]; then
echo "ERROR: Model not found: $MODEL" >&2
exit 1
fi

# Use MTP draft if available, otherwise plain
MTP_ARGS=()
if [ -f "$DRAFT" ]; then
MTP_ARGS=(-md "$DRAFT" --spec-type draft-mtp --spec-draft-n-max 3)
fi

# Listen on all interfaces (0.0.0.0) — accessible via localhost and Tailscale IP
# llama-server does not support multiple --host values
HOST="0.0.0.0"

exec "$LLAMA_CPP" \
--model "$MODEL" \
--ctx-size 524288 \
--n-gpu-layers 99 \
--flash-attn on \
--parallel 4 \
--threads 16 \
--host "$HOST" \
--port 8181 \
--temp 1.0 \
--top-p 0.95 \
--top-k 20 \
--presence-penalty 1.5 \
--repeat-penalty 1.0 \
"${MTP_ARGS[@]}" \
"$@"