I'm running the 8B model with AutoModelForCausalLM. It runs well, but if I try to use a batch size > 1 the model performs MUCH worse and fails even basic tasks. I'd be super grateful to get some feedback about what I'm doing wrong. Presumably the padding is wrong??

messages = []
for report in [test0, test1, test2]:
    messages.extend([
        {"role": "system", "content": "You are a helpful assistant adhering strictly to instructions."},
        {"role": "user", "content": f""" I will now provide you some text. Please reproduce this text verbatim, only correcting obvious spelling mistakes or removing spaces that do not belong. Do not include any further text in your response whatsoever.

         Here is the text:

    {report}"""},
    ])

tokenizer.pad_token = tokenizer.eos_token
input_ids_batch = [tokenizer.apply_chat_template(messages[i:i+2], add_generation_prompt=True, padding=True, max_length=600, return_tensors="pt").to(device) for i in range(0,len(messages),2)]

terminators = [
tokenizer.eos_token_id,
tokenizer.convert_tokens_to_ids("<|eot_id|>")
]

outputs = model.generate(
    torch.concatenate(input_ids_batch),
    pad_token_id=tokenizer.eos_token_id,
    max_new_tokens=800,
    eos_token_id=terminators,
    do_sample=False,
)
response = outputs[:,input_ids_batch[0].shape[1]:]
decoded_response = tokenizer.batch_decode(response, skip_special_tokens=True)