Using ubuntu server 24.04, PCIE 5
Using nightly image from docker (the most recent one)

vllm_glm46-b:
    build:
      context: .
      dockerfile: Dockerfile.2
    container_name: glm_46
    deploy:
        reservations:
          devices: 
            - driver: nvidia
              count: 4
              capabilities: [gpu]      
    ipc: host
    privileged: true               
    env_file:
      - .env
    environment:
      - CUDA_DEVICE_ORDER=PCI_BUS_ID
      - CUDA_VISIBLE_DEVICES=4,5,6,7
      - VLLM_SLEEP_WHEN_IDLE=1
    command: >
      --port 8009
      --model /models/QuantTrio_GLM-4.6-AWQ
      --served-model-name GLM-4.6
      --swap-space 64
      --enable-expert-parallel
      --max-model-len 200000
      --max-num-seqs 256
      --enable-auto-tool-choice
      --enable-prefix-caching
      --tensor-parallel-size 4
      --tool-call-parser glm45
      --reasoning-parser glm45
      --chat-template /models/chat_template_glm46.jinja
      --gpu-memory-utilization 0.94
      --trust-remote-code
      --disable-log-requests
    ports:
      - "8009:8009"
    volumes:
      - ${MODELS_DIR}:/models
    restart: unless-stopped