From aa5e50f1d2b6ef88fc7f494a9322d8aa60997b3c Mon Sep 17 00:00:00 2001 From: Calvin Montgomery Date: Sat, 27 Dec 2014 01:39:30 -0500 Subject: [PATCH 1/4] Cytubefilters, part 1 --- lib/channel/chat.js | 4 +- lib/channel/filters.js | 267 ++++++++++++++++++++--------------------- 2 files changed, 133 insertions(+), 138 deletions(-) diff --git a/lib/channel/chat.js b/lib/channel/chat.js index 5cfa0c1c..de63b83c 100644 --- a/lib/channel/chat.js +++ b/lib/channel/chat.js @@ -315,7 +315,7 @@ ChatModule.prototype.filterMessage = function (msg) { /* substring is a URL */ if (convertLinks && parts[j].match(link)) { var original = parts[j]; - parts[j] = filters.exec(parts[j], { filterlinks: true }); + parts[j] = filters.filter(parts[j], true); /* no filters changed the URL, apply link filter */ if (parts[j] === original) { @@ -325,7 +325,7 @@ ChatModule.prototype.filterMessage = function (msg) { } else { /* substring is not a URL */ - parts[j] = filters.exec(parts[j], { filterlinks: false }); + parts[j] = filters.filter(parts[j], false); } } diff --git a/lib/channel/filters.js b/lib/channel/filters.js index 95e93380..8b47c8a1 100644 --- a/lib/channel/filters.js +++ b/lib/channel/filters.js @@ -1,120 +1,7 @@ +var FilterList = require('cytubefilters'); var ChannelModule = require("./module"); var XSS = require("../xss"); - -function ChatFilter(name, regex, flags, replace, active, filterlinks) { - this.name = name; - this.source = regex; - this.flags = flags; - this.regex = new RegExp(this.source, flags); - this.replace = replace; - this.active = active === false ? false : true; - this.filterlinks = filterlinks || false; -} - -ChatFilter.prototype = { - pack: function () { - return { - name: this.name, - source: this.source, - flags: this.flags, - replace: this.replace, - active: this.active, - filterlinks: this.filterlinks - }; - }, - - exec: function (str) { - return str.replace(this.regex, this.replace); - } -}; - -function FilterList(defaults) { - if (!defaults) { - defaults = []; - } - - this.filters = defaults.map(function (f) { - return new ChatFilter(f.name, f.source, f.flags, f.replace, f.active, f.filterlinks); - }); -} - -FilterList.prototype = { - pack: function () { - return this.filters.map(function (f) { return f.pack(); }); - }, - - importList: function (filters) { - this.filters = Array.prototype.slice.call(filters); - }, - - updateFilter: function (filter) { - if (!filter.name) { - filter.name = filter.source; - } - - var found = false; - for (var i = 0; i < this.filters.length; i++) { - if (this.filters[i].name === filter.name) { - found = true; - this.filters[i] = filter; - break; - } - } - - /* If no filter was updated, add a new one */ - if (!found) { - this.filters.push(filter); - } - }, - - removeFilter: function (filter) { - var found = false; - for (var i = 0; i < this.filters.length; i++) { - if (this.filters[i].name === filter.name) { - this.filters.splice(i, 1); - break; - } - } - }, - - moveFilter: function (from, to) { - if (from < 0 || to < 0 || - from >= this.filters.length || to >= this.filters.length) { - return false; - } - - var f = this.filters[from]; - /* Offset from/to indexes to account for the fact that removing - an element changes the position of one of them. - - I could have just done a swap, but it's already implemented this way - and it works. */ - to = to > from ? to + 1 : to; - from = to > from ? from : from + 1; - - this.filters.splice(to, 0, f); - this.filters.splice(from, 1); - return true; - }, - - exec: function (str, opts) { - if (!opts) { - opts = {}; - } - - this.filters.forEach(function (f) { - if (opts.filterlinks && !f.filterlinks) { - return; - } - - if (f.active) { - str = f.exec(str); - } - }); - - return str; - } -}; +var Logger = require("../logger"); function validateFilter(f) { if (typeof f.source !== "string" || typeof f.flags !== "string" || @@ -136,17 +23,39 @@ function validateFilter(f) { return false; } - var filter = new ChatFilter(f.name, f.source, f.flags, f.replace, - Boolean(f.active), Boolean(f.filterlinks)); + var filter = { + name: f.name, + source: f.source, + replace: fixReplace(f.replace), + flags: f.flags, + active: !!f.active, + filterlinks: !!f.filterlinks + }; + return filter; } +function fixReplace(replace) { + return replace.replace(/\$(\d)/g, '\\$1'); +} + +function makeDefaultFilter(name, source, flags, replace) { + return { + name: name, + source: source, + flags: flags, + replace: fixReplace(replace), + active: true, + filterlinks: false + }; +} + const DEFAULT_FILTERS = [ - new ChatFilter("monospace", "`(.+?)`", "g", "$1"), - new ChatFilter("bold", "\\*(.+?)\\*", "g", "$1"), - new ChatFilter("italic", "_(.+?)_", "g", "$1"), - new ChatFilter("strike", "~~(.+?)~~", "g", "$1"), - new ChatFilter("inline spoiler", "\\[sp\\](.*?)\\[\\/sp\\]", "ig", "$1") + makeDefaultFilter("monospace", "`(.+?)`", "g", "$1"), + makeDefaultFilter("bold", "\\*(.+?)\\*", "g", "$1"), + makeDefaultFilter("italic", "_(.+?)_", "g", "$1"), + makeDefaultFilter("strike", "~~(.+?)~~", "g", "$1"), + makeDefaultFilter("inline spoiler", "\\[sp\\](.*?)\\[\\/sp\\]", "ig", "$1") ]; function ChatFilterModule(channel) { @@ -161,7 +70,21 @@ ChatFilterModule.prototype.load = function (data) { for (var i = 0; i < data.filters.length; i++) { var f = validateFilter(data.filters[i]); if (f) { - this.filters.updateFilter(f); + try { + this.filters.updateFilter(f); + } catch (e) { + if (e.message.match(/does not exist/i)) { + try { + this.filters.addFilter(f); + } catch (e) { + Logger.errlog.log("Filter load failed: " + + JSON.stringify(f) + " c:" + this.channel.name); + } + } else { + Logger.errlog.log("Filter load failed: " + + JSON.stringify(f) + " c:" + this.channel.name); + } + } } } } @@ -173,11 +96,12 @@ ChatFilterModule.prototype.save = function (data) { ChatFilterModule.prototype.packInfo = function (data, isAdmin) { if (isAdmin) { - data.chatFilterCount = this.filters.filters.length; + data.chatFilterCount = this.filters.length; } }; ChatFilterModule.prototype.onUserPostJoin = function (user) { + user.socket.on("addFilter", this.handleAddFilter.bind(this, user)); user.socket.on("updateFilter", this.handleUpdateFilter.bind(this, user)); user.socket.on("importFilters", this.handleImportFilters.bind(this, user)); user.socket.on("moveFilter", this.handleMoveFilter.bind(this, user)); @@ -195,6 +119,41 @@ ChatFilterModule.prototype.sendChatFilters = function (users) { }); }; +ChatFilterModule.prototype.handleAddFilter = function (user, data) { + if (typeof data !== "object") { + return; + } + + if (!this.channel.modules.permissions.canEditFilters(user)) { + return; + } + + data = validateFilter(data); + if (!data) { + return; + } + + try { + this.filters.addFilter(data); + } catch (e) { + user.socket.emit("errorMsg", { + msg: "Filter add failed: " + e.message, + alert: true + }); + return; + } + var chan = this.channel; + chan.users.forEach(function (u) { + if (chan.modules.permissions.canEditFilters(u)) { + u.socket.emit("updateChatFilter", data); + } + }); + + chan.logger.log("[mod] " + user.getName() + " added filter: " + data.name + " -> " + + "s/" + data.source + "/" + data.replace + "/" + data.flags + " active: " + + data.active + ", filterlinks: " + data.filterlinks); +}; + ChatFilterModule.prototype.handleUpdateFilter = function (user, data) { if (typeof data !== "object") { return; @@ -204,13 +163,25 @@ ChatFilterModule.prototype.handleUpdateFilter = function (user, data) { return; } - var f = validateFilter(data); - if (!f) { + data = validateFilter(data); + if (!data) { + return; + } + + try { + this.filters.updateFilter(data); + } catch (e) { + if (e.message.match(/filter to be updated does not exist/i)) { + this.handleAddFilter(user, data); + } else { + user.socket.emit("errorMsg", { + msg: "Filter update failed: " + e.message, + alert: true + }); + } return; } - data = f.pack(); - this.filters.updateFilter(f); var chan = this.channel; chan.users.forEach(function (u) { if (chan.modules.permissions.canEditFilters(u)) { @@ -218,9 +189,9 @@ ChatFilterModule.prototype.handleUpdateFilter = function (user, data) { } }); - chan.logger.log("[mod] " + user.getName() + " updated filter: " + f.name + " -> " + - "s/" + f.source + "/" + f.replace + "/" + f.flags + " active: " + - f.active + ", filterlinks: " + f.filterlinks); + chan.logger.log("[mod] " + user.getName() + " updated filter: " + data.name + " -> " + + "s/" + data.source + "/" + data.replace + "/" + data.flags + " active: " + + data.active + ", filterlinks: " + data.filterlinks); }; ChatFilterModule.prototype.handleImportFilters = function (user, data) { @@ -234,9 +205,17 @@ ChatFilterModule.prototype.handleImportFilters = function (user, data) { return; } - this.filters.importList(data.map(validateFilter).filter(function (f) { - return f !== false; - })); + try { + this.filters = new FilterList(data.map(validateFilter).filter(function (f) { + return f !== false; + })); + } catch (e) { + user.socket.emit("errorMsg", { + msg: "Filter import failed: " + e.message, + alert: true + }); + return; + } this.channel.logger.log("[mod] " + user.getName() + " imported the filter list"); this.sendChatFilters(this.channel.users); @@ -255,7 +234,15 @@ ChatFilterModule.prototype.handleRemoveFilter = function (user, data) { return; } - this.filters.removeFilter(data); + try { + this.filters.removeFilter(data); + } catch (e) { + user.socket.emit("errorMsg", { + msg: "Filter removal failed: " + e.message, + alert: true + }); + return; + } var chan = this.channel; chan.users.forEach(function (u) { if (chan.modules.permissions.canEditFilters(u)) { @@ -278,7 +265,15 @@ ChatFilterModule.prototype.handleMoveFilter = function (user, data) { return; } - this.filters.moveFilter(data.from, data.to); + try { + this.filters.moveFilter(data.from, data.to); + } catch (e) { + user.socket.emit("errorMsg", { + msg: "Filter move failed: " + e.message, + alert: true + }); + return; + } }; ChatFilterModule.prototype.handleRequestChatFilters = function (user) { From 25eba6ab2bb693c36a1f86066087163072a204e5 Mon Sep 17 00:00:00 2001 From: Calvin Montgomery Date: Sun, 28 Dec 2014 11:12:37 -0500 Subject: [PATCH 2/4] Improve filter handling code --- lib/channel/filters.js | 113 ++++++++++++++++++++++++----------------- www/js/ui.js | 20 +++----- www/js/util.js | 10 ++-- 3 files changed, 77 insertions(+), 66 deletions(-) diff --git a/lib/channel/filters.js b/lib/channel/filters.js index 8b47c8a1..fbf1ab2c 100644 --- a/lib/channel/filters.js +++ b/lib/channel/filters.js @@ -1,26 +1,34 @@ -var FilterList = require('cytubefilters'); +var FilterList = require("cytubefilters"); var ChannelModule = require("./module"); var XSS = require("../xss"); var Logger = require("../logger"); +/* + * Converts JavaScript-style replacements ($1, $2, etc.) with + * PCRE-style (\1, \2, etc.) + */ +function fixReplace(replace) { + return replace.replace(/\$(\d)/g, "\\$1"); +} + function validateFilter(f) { if (typeof f.source !== "string" || typeof f.flags !== "string" || typeof f.replace !== "string") { - return false; + return null; } if (typeof f.name !== "string") { f.name = f.source; } - f.replace = f.replace.substring(0, 1000); + f.replace = fixReplace(f.replace.substring(0, 1000)); f.replace = XSS.sanitizeHTML(f.replace); f.flags = f.flags.substring(0, 4); try { - new RegExp(f.source, f.flags); + FilterList.checkValidRegex(f.source); } catch (e) { - return false; + return null; } var filter = { @@ -35,58 +43,47 @@ function validateFilter(f) { return filter; } -function fixReplace(replace) { - return replace.replace(/\$(\d)/g, '\\$1'); -} - function makeDefaultFilter(name, source, flags, replace) { return { name: name, source: source, flags: flags, - replace: fixReplace(replace), + replace: replace, active: true, filterlinks: false }; } const DEFAULT_FILTERS = [ - makeDefaultFilter("monospace", "`(.+?)`", "g", "$1"), - makeDefaultFilter("bold", "\\*(.+?)\\*", "g", "$1"), - makeDefaultFilter("italic", "_(.+?)_", "g", "$1"), - makeDefaultFilter("strike", "~~(.+?)~~", "g", "$1"), - makeDefaultFilter("inline spoiler", "\\[sp\\](.*?)\\[\\/sp\\]", "ig", "$1") + makeDefaultFilter("monospace", "`(.+?)`", "g", "\\1"), + makeDefaultFilter("bold", "\\*(.+?)\\*", "g", "\\1"), + makeDefaultFilter("italic", "_(.+?)_", "g", "\\1"), + makeDefaultFilter("strike", "~~(.+?)~~", "g", "\\1"), + makeDefaultFilter("inline spoiler", "\\[sp\\](.*?)\\[\\/sp\\]", "ig", + "\\1") ]; function ChatFilterModule(channel) { ChannelModule.apply(this, arguments); - this.filters = new FilterList(DEFAULT_FILTERS); + this.filters = new FilterList(); } ChatFilterModule.prototype = Object.create(ChannelModule.prototype); ChatFilterModule.prototype.load = function (data) { if ("filters" in data) { - for (var i = 0; i < data.filters.length; i++) { - var f = validateFilter(data.filters[i]); - if (f) { - try { - this.filters.updateFilter(f); - } catch (e) { - if (e.message.match(/does not exist/i)) { - try { - this.filters.addFilter(f); - } catch (e) { - Logger.errlog.log("Filter load failed: " + - JSON.stringify(f) + " c:" + this.channel.name); - } - } else { - Logger.errlog.log("Filter load failed: " + - JSON.stringify(f) + " c:" + this.channel.name); - } - } - } + var filters = data.filters.map(validateFilter).filter(function (f) { + return f !== null; + }); + try { + this.filters = new FilterList(filters); + } catch (e) { + Logger.errlog.log("Filter load failed: " + e + " (channel:" + + this.channel.name); + this.channel.logger.log("Failed to load filters: " + e); } + } else { + this.filters = new FilterList(DEFAULT_FILTERS); } }; @@ -128,6 +125,16 @@ ChatFilterModule.prototype.handleAddFilter = function (user, data) { return; } + try { + FilterList.checkValidRegex(data.source); + } catch (e) { + user.socket.emit("errorMsg", { + msg: "Invalid regex: " + e.message, + alert: true + }); + return; + } + data = validateFilter(data); if (!data) { return; @@ -142,6 +149,9 @@ ChatFilterModule.prototype.handleAddFilter = function (user, data) { }); return; } + + user.socket.emit("addFilterSuccess"); + var chan = this.channel; chan.users.forEach(function (u) { if (chan.modules.permissions.canEditFilters(u)) { @@ -150,8 +160,8 @@ ChatFilterModule.prototype.handleAddFilter = function (user, data) { }); chan.logger.log("[mod] " + user.getName() + " added filter: " + data.name + " -> " + - "s/" + data.source + "/" + data.replace + "/" + data.flags + " active: " + - data.active + ", filterlinks: " + data.filterlinks); + "s/" + data.source + "/" + data.replace + "/" + data.flags + + " active: " + data.active + ", filterlinks: " + data.filterlinks); }; ChatFilterModule.prototype.handleUpdateFilter = function (user, data) { @@ -163,6 +173,16 @@ ChatFilterModule.prototype.handleUpdateFilter = function (user, data) { return; } + try { + FilterList.checkValidRegex(data.source); + } catch (e) { + user.socket.emit("errorMsg", { + msg: "Invalid regex: " + e.message, + alert: true + }); + return; + } + data = validateFilter(data); if (!data) { return; @@ -171,14 +191,10 @@ ChatFilterModule.prototype.handleUpdateFilter = function (user, data) { try { this.filters.updateFilter(data); } catch (e) { - if (e.message.match(/filter to be updated does not exist/i)) { - this.handleAddFilter(user, data); - } else { - user.socket.emit("errorMsg", { - msg: "Filter update failed: " + e.message, - alert: true - }); - } + user.socket.emit("errorMsg", { + msg: "Filter update failed: " + e.message, + alert: true + }); return; } @@ -190,8 +206,8 @@ ChatFilterModule.prototype.handleUpdateFilter = function (user, data) { }); chan.logger.log("[mod] " + user.getName() + " updated filter: " + data.name + " -> " + - "s/" + data.source + "/" + data.replace + "/" + data.flags + " active: " + - data.active + ", filterlinks: " + data.filterlinks); + "s/" + data.source + "/" + data.replace + "/" + data.flags + + " active: " + data.active + ", filterlinks: " + data.filterlinks); }; ChatFilterModule.prototype.handleImportFilters = function (user, data) { @@ -207,7 +223,7 @@ ChatFilterModule.prototype.handleImportFilters = function (user, data) { try { this.filters = new FilterList(data.map(validateFilter).filter(function (f) { - return f !== false; + return f !== null; })); } catch (e) { user.socket.emit("errorMsg", { @@ -249,6 +265,7 @@ ChatFilterModule.prototype.handleRemoveFilter = function (user, data) { u.socket.emit("deleteChatFilter", data); } }); + this.channel.logger.log("[mod] " + user.getName() + " removed filter: " + data.name); }; diff --git a/www/js/ui.js b/www/js/ui.js index aa0ff3b9..17016a82 100644 --- a/www/js/ui.js +++ b/www/js/ui.js @@ -621,14 +621,7 @@ $("#cs-chatfilters-newsubmit").click(function () { "match."); } - try { - new RegExp(regex, flags); - } catch (e) { - alert("Regex error: " + e); - return; - } - - socket.emit("updateFilter", { + socket.emit("addFilter", { name: name, source: regex, flags: flags, @@ -636,10 +629,13 @@ $("#cs-chatfilters-newsubmit").click(function () { active: true }); - $("#cs-chatfilters-newname").val(""); - $("#cs-chatfilters-newregex").val(""); - $("#cs-chatfilters-newflags").val(""); - $("#cs-chatfilters-newreplace").val(""); + socket.once("addFilterSuccess", function () { + console.log("addFilterSuccess"); + $("#cs-chatfilters-newname").val(""); + $("#cs-chatfilters-newregex").val(""); + $("#cs-chatfilters-newflags").val(""); + $("#cs-chatfilters-newreplace").val(""); + }); }); $("#cs-emotes-newsubmit").click(function () { diff --git a/www/js/util.js b/www/js/util.js index 8e625b05..0ae644fd 100644 --- a/www/js/util.js +++ b/www/js/util.js @@ -2304,14 +2304,12 @@ function formatCSChatFilterList() { f.flags = flags.val(); f.replace = replace.val(); f.filterlinks = filterlinks.prop("checked"); - try { - new RegExp(f.source, f.flags); - } catch (e) { - alert("Invalid regex: " + e); - } socket.emit("updateFilter", f); - reset(); + socket.once("updateFilterSuccess", function () { + console.log("updateFilterSuccess"); + reset(); + }); }); control.data("editor", tr2); From 058b24323d9a641e7a8ec28113782f8c20abc08c Mon Sep 17 00:00:00 2001 From: Calvin Montgomery Date: Sun, 28 Dec 2014 19:07:39 -0500 Subject: [PATCH 3/4] Add cytubefilters to package.json --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index adfe2de9..eebad5ca 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "body-parser": "^1.6.5", "compression": "^1.2.0", "cookie-parser": "^1.3.2", + "cytubefilters": "git://github.com/calzoneman/cytubefilters#a5a99642", "express": "^4.8.5", "express-minify": "0.0.11", "jade": "^1.5.0", From 4319111f47198ab69829c2450e4a8fbd86686357 Mon Sep 17 00:00:00 2001 From: Calvin Montgomery Date: Sun, 28 Dec 2014 19:09:41 -0500 Subject: [PATCH 4/4] Remove console.log --- www/js/ui.js | 1 - www/js/util.js | 1 - 2 files changed, 2 deletions(-) diff --git a/www/js/ui.js b/www/js/ui.js index 17016a82..3609faaa 100644 --- a/www/js/ui.js +++ b/www/js/ui.js @@ -630,7 +630,6 @@ $("#cs-chatfilters-newsubmit").click(function () { }); socket.once("addFilterSuccess", function () { - console.log("addFilterSuccess"); $("#cs-chatfilters-newname").val(""); $("#cs-chatfilters-newregex").val(""); $("#cs-chatfilters-newflags").val(""); diff --git a/www/js/util.js b/www/js/util.js index 0ae644fd..e175cdfc 100644 --- a/www/js/util.js +++ b/www/js/util.js @@ -2307,7 +2307,6 @@ function formatCSChatFilterList() { socket.emit("updateFilter", f); socket.once("updateFilterSuccess", function () { - console.log("updateFilterSuccess"); reset(); }); });