Сделал так
let timerUserInactivity,
flag = true;
stage.addEventListener('mousemove', userInactivity, false);
btn.onclick = function() {
flag = false;
stage.removeEventListener('mousemove', userInactivity, false);
}
function userInactivity() {
window.clearTimeout(timerUserInactivity);
timerUserInactivity = setTimeout(function() {
console.log('test');
if(flag) {
userInactivity();
}
}, 1000);
}