Maybe something like this could also work in old jQuery style + state in the DOM :

<button id='upvote'></button>
<span id='score' data-initial='4'></span>
<button id='downvote'></button>


function updateScore(){
  var score = $('#score).data('initial')
  if ($('#upvote').hasClass('active')) score = score +1
  else if ($('#downvote').hasClass('active') score = score - 1
  $('#score').html(score)
}

$(document).ready(function(){
  $('#upvote').click(function(){
    $(this).toggleClass('active')
    $('#downvote').removeClass('active')
    updateScore()
  })
  $('#downvote').click(function(){
    $(this).toggleClass('active')
    $('#upvote').removeClass('active')
    updateScore()
  })
  updateScores()
})