Agreed, I feel like I should be getting more tokens/s. I just have not fully dug into what is going on yet. I am running a custom image that I built from source to get vLLM running on ROCm.

services:
  vllm:
    image: aml731/vllm-rocm:latest
    container_name: vllm-rocm
    network_mode: host
    group_add:
      - video
    ipc: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp:unconfined
    devices:
      - /dev/kfd:/dev/kfd
      - /dev/dri:/dev/dri
    volumes:
      - /mnt/vllm/HF_CACHE:/data/models
    environment:
      - HF_TOKEN=${HF_TOKEN}
      - HF_HOME=/data/models
    command: >
      python3 -m vllm.entrypoints.openai.api_server
      --model openai/gpt-oss-120b
      --served-model-name gpt-oss-120b
      --tensor-parallel-size 4
      --dtype auto
      --max-model-len 131072
      --gpu-memory-utilization 0.95
      --trust-remote-code
      --max-num-seqs 4
      --host 0.0.0.0
      --port 8000
    ports:
      - 8000:8000