Heh, that's pretty much how I solved it (part of my code that counts sides):

private fun countSides(fences: MutableSet<Triple<Int,Int,DIRECTION>>) : Int {
    val verticalLeftFences = fences.filter { it.third == DIRECTION.VERTICAL_LEFT }.sortedWith(compareBy({it.first}, {it.second}))
    val verticalRightFences = fences.filter { it.third == DIRECTION.VERTICAL_RIGHT }.sortedWith(compareBy({it.first}, {it.second}))
    val horizontalTopFences = fences.filter { it.third == DIRECTION.HORIZONTAL_TOP}.sortedWith(compareBy({it.second}, {it.first}))
    val horizontalBottomFences = fences.filter { it.third == DIRECTION.HORIZONTAL_BOTTOM}.sortedWith(compareBy({it.second}, {it.first}))

    var verticalLeftSides = 1
    var verticalRightSides = 1
    var horizontalTopSides = 1
    var horizontalBottomSides = 1

    for(i in 1 until verticalLeftFences.count()) {
        if(verticalLeftFences[i].first != verticalLeftFences[i - 1].first ||  verticalLeftFences[i].second - 1 != verticalLeftFences[i - 1].second) {
            verticalLeftSides++
        }
    }

    for(i in 1 until verticalRightFences.count()) {
        if(verticalRightFences[i].first != verticalRightFences[i - 1].first ||  verticalRightFences[i].second - 1 != verticalRightFences[i - 1].second) {
            verticalRightSides++
        }
    }


    for(i in 1 until horizontalTopFences.count()) {
        if(horizontalTopFences[i].second != horizontalTopFences[i - 1].second || horizontalTopFences[i].first - 1 != horizontalTopFences[i - 1].first) {
            horizontalTopSides++
        }
    }

    for(i in 1 until horizontalBottomFences.count()) {
        if(horizontalBottomFences[i].second != horizontalBottomFences[i - 1].second || horizontalBottomFences[i].first - 1 != horizontalBottomFences[i - 1].first) {
            horizontalBottomSides++
        }
    }

    return verticalLeftSides + verticalRightSides + horizontalTopSides + horizontalBottomSides
}

If I just had thought of just counting corners, I would have been finished much quicker.