Edit in JSFiddle

var insertsList = [
    ["(", ")"],
    ["{", "}"],
    ["[", "]"],
    ["<", ">"]
];

document.onkeypress = function (event) {
    var node = event.target;

    if (node.tagName === "TEXTAREA") handleKeyPress.call(node, event);
};

/*assumes node to be textarea*/
function handleKeyPress(event) {
    var keyCode = event.keyCode; // step 1

    var charTyped = String.fromCharCode(keyCode), // step 2
        index = searchInsertsList(charTyped); // step 3

    if (index !== -1) { // step 4 starts
        event.preventDefault(); // prevent the bracket from getting inserted
        // we will insert it later

        // create text
        var textArray = insertsList[index],
            text = textArray[0] + textArray[1],
            val = this.value;

        var nSS = this.selectionStart,
            nSE = this.selectionEnd,
            curPos = nSS + 1; // in the middle of the two characters

        // set the value
        this.value = val.substring(0, nSS) + text + val.substring(nSE);

        this.selectionStart = this.selectionEnd = curPos;
    }

    if (charTyped === "=") { // step 1
        // step 2 begins
        var curPos = this.selectionStart,
            value = this.value,
            expr = "";

        // build our math expression string
        for (var i = curPos - 1; value[i] !== "["; i--)
            expr = value[i] + expr;

        // no end brackets are present.
        // user didn’t want to do Mathomania; user wanted literal `=` only
        // position of firstBracket and lastBracket in `[[math_expr=]]`
        var firstBracket = i - 1,
            lastBracket = curPos + 1;
        
        if (value[lastBracket - 1] + value[lastBracket] !== "]]") {
            return;
        }
        
        // do not insert `=` sign
        event.preventDefault();
        
        // evaluate expression (step 3) safely
        // replacing any non-digit non-math-operator characters
        // with nothing (empty string)
        var result = "";
        try {
            result = eval(expr.replace(/[^\d\+\-\\\*]/g, ""));
        } catch (error) {}
        
        // step 4
        // get text preceding the first bracket, concatenate it with result
        // then append text after the lastBracket
        this.value = value.substring(0, firstBracket) + result + value.substring(lastBracket + 1);

        // set caret position immediately after result
        this.selectionStart = this.selectionEnd = firstBracket + result.toString().length;
    }
}

function searchInsertsList(charTyped) {
    for (var i = 0, len = insertsList.length; i < len; i++) {
        // character found? return index
        if (insertsList[i][0] === charTyped) return i;
    }

    return -1; // character not found
}

<textarea></textarea>