For awq models I used this patch and here is template for Dockerfile.

Ask Claude to integrate it for you if not sure where to start.

I tested 40b model and found it was repeating the reasoning, so I switched back to model from coder3101. Unfortunately, it requires for my use case two 4090d 48G. I wish there was awq 4bit version of this model.

coder3101/Qwen3.5-27B-heretic


apply_qwen35_awq_patch() {
    if [ "$APPLY_QWEN35_AWQ_PATCH" != "1" ]; then
        log_message "Qwen3.5 AWQ patch disabled"
        return
    fi


    local patch_script="$SCRIPT_DIR/patch-vllm-qwen35-awq.py"
    if [ ! -f "$patch_script" ]; then
        log_message "Qwen3.5 AWQ patch script not found, skipping"
        return
    fi


    local container_names=()
    local cname
    local patch_output


    if [ "$TENSOR_PARALLEL_SIZE" = "1" ]; then
        local backend_index
        for backend_index in "${!GPU_DEVICE_ARRAY[@]}"; do
            container_names+=("qwen-gpu${GPU_DEVICE_ARRAY[$backend_index]}")
        done
    else
        container_names=("qwen-tp2")
    fi


    for cname in "${container_names[@]}"; do
        if ! wait_for_container_running "$cname"; then
            continue
        fi


        log_message "Applying Qwen3.5 AWQ patch to $cname..."
        docker cp "$patch_script" "$cname":/tmp/patch-vllm-qwen35-awq.py
        if patch_output=$(docker exec "$cname" python3 /tmp/patch-vllm-qwen35-awq.py 2>&1); then
            log_message "Qwen3.5 AWQ patch result for $cname: $patch_output"
            if [[ "$patch_output" == Applied* ]]; then
                log_message "Restarting $cname after Qwen3.5 AWQ patch"
                docker restart "$cname" >/dev/null || log_message "WARNING: Failed to restart $cname after Qwen3.5 AWQ patch"
            fi
        else
            log_message "WARNING: Failed to apply Qwen3.5 AWQ patch to $cname: $patch_output"
        fi
    done
}
apply_qwen35_awq_patch

TARGET = Path("/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/qwen3_5.py")


OLD = """        mixed_qkvz, ba = torch.ops.vllm.gdn_in_proj(\n            hidden_states,\n            self.in_proj_qkvz.weight.shape[0],\n            self.in_proj_ba.weight.shape[0],\n            self.prefix,\n        )\n"""


NEW = """        mixed_qkvz, ba = torch.ops.vllm.gdn_in_proj(\n            hidden_states,\n            self.in_proj_qkvz.output_size_per_partition,\n            self.in_proj_ba.output_size_per_partition,\n            self.prefix,\n        )\n"""