Вариант для (){}[]
function balance(a) {
for (var d = /(\u005B|\u0028|\u007B)[^\u005B\u0028\u007B]*?$/,
e = {
"(": /\u0028[^\u007D\u005D]*?\u0029/,
"[": /\u005B[^\u0029\u007D]*?\u005D/,
"{": /\u007B[^\u0029\u005D]*?\u007D/
}, b, c = !0; c;) b = a, a = a.replace(d, function (a, b) {
return a.replace(e[b], "")
}), b == a && (c = !1);
return !/[\u005B\u005D\u0028\u0029\u007B\u007D]/.test(a)
};
alert(balance("{}({}123(45))")+"\n"+balance("{}{}1234][5")+"\n"+balance("<><(([]){}<[{{{{}}[[]]}}]>)>"))