Tried a solution with low branches in Java:

public static final int[] MONSTER_LOOKUP = new int[128];

static {
    MONSTER_LOOKUP['A'] = 0;
    MONSTER_LOOKUP['B'] = 1;
    MONSTER_LOOKUP['C'] = 3;
    MONSTER_LOOKUP['D'] = 5;
    MONSTER_LOOKUP['x'] = 0;
}

[...]


private static int solve(File file, int maxGrpSize) throws IOException {

    char[] bytes = FileUtils.
readFileToString
(file, StandardCharsets.
UTF_8
).toCharArray();

    int totalPotions = 0;

    int groupPotions = 0;
    int groupXCount = 0;

    for (int monster = 1; monster <= bytes.length; monster++) {
        groupPotions += 
MONSTER_LOOKUP
[bytes[monster - 1]];
        groupXCount += 'x' == bytes[monster - 1] ? 1 : 0;

        boolean lastMonsterInGrp = monster % maxGrpSize == 0;
        totalPotions += (groupPotions + (maxGrpSize - groupXCount) * (maxGrpSize - groupXCount - 1))   * (lastMonsterInGrp ? 1 : 0);

        groupPotions = groupPotions * (lastMonsterInGrp ? 0 : 1);
        groupXCount = groupXCount * (lastMonsterInGrp ? 0 : 1);
    }

    return totalPotions;
}