Edit in JSFiddle

$("input[type=checkbox]").click(function(){
    var countchecked = $("input[type=checkbox]:checked").length;

    if(countchecked >= 3) {
        $('input[type=checkbox]').not(':checked').attr("disabled",true);
    }else{
        $('input[type=checkbox]').not(':checked').attr("disabled",false);
    }
});

<input type="checkbox" value="1">
<input type="checkbox" value="1">
<input type="checkbox" value="1">
<input type="checkbox" value="1">
<input type="checkbox" value="1">
<input type="checkbox" value="1">
<input type="checkbox" value="1">
<input type="checkbox" value="1">
<input type="checkbox" value="1">
<input type="checkbox" value="1">
<input type="checkbox" value="1">
<input type="checkbox" value="1">
<input type="checkbox" value="1">
<input type="checkbox" value="1">

label {display:block;}