Edit in JSFiddle

$("#hello").removeClass (function (index, css) {
    return (css.match (/\bcolor-\S+/g) || []).join(' ');
});

.color-red{color:red;}

<div id="hello" class="color-red color-brown foo bar">Hello</div>