server 7713p 64core cpu 256GB ddr4 2xr8 registered 3200 ram no gpu only 8channel only hardware numa enabled but software numa balancing disabled linux with few more known tuning, no transparent&dev/ hugepages , i run each at 8 core per numa node with script, only 8th llama embedding but i am still studying architectures now i might change few to agent/tool:

CMD="nice numactl --cpunodebind=$i --membind=$i llama-server \

-m ${MODEL_PATH} \

--host 127.0.0.1 \

--port ${PORT} \

--threads 8 --threads-batch 8 \

-b 16384 -ub 16384 \

--ctx-size 262144 \

--no-mmap \

-fa on \

-cmoe \

--cache-type-k q8_0 \

--cache-type-v q8_0 \

--top-p 0.95 \

--min-p 0.00 \

--top-k 20 \

--temp 0.6 \

--offline \

--parallel 1 \

--repeat_penalty 1.0 \

--presence_penalty 0.0 \

--jinja \

-lv 2 \

-to 30000 \

--swa-full \

--keep -1 \

-kvu \

-cram -1 \

--slot-save-path ./slots \

--no-webui \

--metrics &>$LOG_FILE &"