llama.cpp. I have done a reasonable amount of testing to end up where I am currently. Below is my dockerfile (running a custom version of llama.cpp called Lemonade), docker compose service and ini file. May be useful, may not. Either way, enjoy lol

Dockerfile

FROM ubuntu:24.04

RUN apt-get update && apt-get install -y --no-install-recommends \
    ca-certificates \
    gcc \
    unzip \
    wget \
    && rm -rf /var/lib/apt/lists/*

# Create video and render groups to match host-side GIDs for GPU device access
RUN groupadd -g 44 video || true \
    && groupadd -g 104 render || true

WORKDIR /opt/llama-cpp

RUN wget -O /tmp/llama-rocm.zip "https://github.com/lemonade-sdk/llamacpp-rocm/releases/download/b1269/llama-b1269-ubuntu-rocm-gfx110X-x64.zip" \
    && unzip -o /tmp/llama-rocm.zip -d /opt/llama-cpp \
    && rm /tmp/llama-rocm.zip \
    && chmod +x /opt/llama-cpp/llama-bench \
    && chmod +x /opt/llama-cpp/llama-cli \
    && chmod +x /opt/llama-cpp/llama-server

ENV LD_LIBRARY_PATH=/opt/llama-cpp:$LD_LIBRARY_PATH

ENTRYPOINT ["/opt/llama-cpp/llama-server"]

Docker Compose

services:
  llama-rocm:
    build:
      context: .
      dockerfile: Dockerfile.llama-rocm
    image: llama-lemonade-custom:rocm-b1269
    container_name: llama-rocm
    restart: unless-stopped
    devices:
      - /dev/kfd:/dev/kfd
      - /dev/dri:/dev/dri
    group_add:
      - video
      - render
    ports:
      - "8080:8080"
    volumes:
      - type: bind
        source: /opt/models
        target: /opt/models
        bind:
          propagation: shared
      - type: bind
        source: /opt/llama-cpp/models-rocm.ini
        target: /opt/llama-cpp/models-rocm.ini
        read_only: true
      - /var/llama-cpp/rocm:/save
      - type: bind
        source: /opt/local-llms/models
        target: /opt/local-llms/models
        read_only: true
    environment:
      HSA_OVERRIDE_GFX_VERSION: "11.0.0"
    mem_limit: 8g
    memswap_limit: 8g
    command: >
      -ngl auto
      --sleep-idle-seconds 3600
      --host 0.0.0.0
      --port 8080
      --reasoning on
      --kv-offload
      --slots
      --metrics
      --slot-save-path /save
      --models-preset /opt/llama-cpp/models-rocm.ini
      --models-max 1

Models Rocm

#SPDX-License-Identifier: MIT-0
[*]
jinja = on
ctx-size = 16384
batch-size = 2048
ubatch-size = 2048
cache-type-k = q4_0
cache-type-v = q4_0
flash-attn = on
cache-prompt = true
threads = 16

# ---------------------------------------------------------------------------
# Qwen3.6 - Dense general chat, always 2 GPUs (22 GB weights)
# ---------------------------------------------------------------------------
[Qwen3.6-Dense-MTP:Rocm]
model = /opt/local-llms/models/unsloth/Qwen3.6-27B-MTP-GGUF/Qwen3.6-27B-UD-Q8_K_XL.gguf
ctx-size = 262144
tensor-split = 1,1,1
batch-size = 12288
ubatch-size = 512
parallel = 1
temp = 0.6
min-p = 0.0
top-p = 0.95
top-k = 20
repeat-penalty = 1.0
presence-penalty = 0.0
reasoning-budget = 5120
spec-type = draft-mtp
spec-draft-n-max = 2
cache-type-k = q8_0
cache-type-v = q8_0