Merge pull request #380 from calzoneman/tablemerge

Merge channel specific tables into global tables with an indexed channel column

index.js

@@ -47,5 +47,11 @@ function handleLine(line) {
         } else {
             Logger.syslog.log("Failed to invoke GC: node started without --expose-gc");
         }
+    } else if (line === "/delete_old_tables") {
+        require("./lib/database/update").deleteOldChannelTables(function (err) {
+            if (!err) {
+                Logger.syslog.log("Deleted old channel tables");
+            }
+        });
     }
 }

lib/channel/kickban.js

@@ -310,7 +310,7 @@ KickBanModule.prototype.banIP = function (actor, ip, name, reason, cb) {
         return error("You do not have ban permissions on this channel");
     }
 
-    Q.nfcall(Account.rankForIP, ip).then(function (rank) {
+    Q.nfcall(Account.rankForIP, ip, { channel: chan.name }).then(function (rank) {
         if (rank >= actor.account.effectiveRank) {
             throw "You don't have permission to ban IP " + masked;
         }

lib/database/channels.js

@@ -19,28 +19,6 @@ function initTables(name, owner, callback) {
         return;
     }
 
-    tables.createChannelTables(name, db.query.bind(db), function (err) {
-        db.users.getGlobalRank(owner, function (err, rank) {
-            if (err) {
-                callback(err, null);
-                return;
-            }
-
-            rank = Math.max(rank, 5);
-
-            module.exports.setRank(name, owner, rank, function (err) {
-                if (err) {
-                    dropTable("chan_" + name + "_ranks");
-                    dropTable("chan_" + name + "_ranks");
-                    dropTable("chan_" + name + "_library");
-                    callback(err, null);
-                    return;
-                }
-
-                callback(null, true);
-            });
-        });
-    });
 }
 
 module.exports = {
@@ -177,14 +155,21 @@ module.exports = {
                     return;
                 }
 
-                initTables(name, owner, function (err, res) {
+                db.users.getGlobalRank(owner, function (err, rank) {
                     if (err) {
-                        db.query("DELETE FROM `channels` WHERE name=?", [name]);
                         callback(err, null);
                         return;
                     }
-                    callback(null, {
+
-                        name: name
+                    rank = Math.max(rank, 5);
+
+                    module.exports.setRank(name, owner, rank, function (err) {
+                        if (err) {
+                            callback(err, null);
+                            return;
+                        }
+
+                        callback(null, { name: name });
                     });
                 });
             });
@@ -204,41 +189,35 @@ module.exports = {
             return;
         }
 
-        dropTable("chan_" + name + "_ranks", function (err) {
+        db.query("DELETE FROM `channels` WHERE name=?", [name], function (err) {
-            dropTable("chan_" + name + "_bans", function (e2) {
+
-                if (err && e2) {
+            module.exports.deleteBans(name, function (err) {
-                    err += "\n" + e2;
+                if (err) {
-                } else if (e2) {
+                    Logger.errlog.log("Failed to delete bans for " + name + ": " + err);
-                    err = e2;
                 }
-
-                dropTable("chan_" + name + "_library", function (e3) {
-                    if (err && e3) {
-                        err += "\n" + e3;
-                    } else if (e3) {
-                        err = e3;
-                    }
-
-                    db.query("DELETE FROM `channels` WHERE name=?", [name],
-                    function (e4) {
-                        if (err && e4) {
-                            err += "\n" + e4;
-                        } else if (e4) {
-                            err = e4;
-                        }
-
-                        fs.unlink(path.join(__dirname, "..", "..", "chandump", name),
-                                  function (err) {
-                            if (err && err.code !== "ENOENT") {
-                                Logger.errlog.log("Deleting chandump failed:");
-                                Logger.errlog.log(err);
-                            }
-                        });
-
-                        callback(err, !Boolean(err));
-                    });
-                });
             });
+
+            module.exports.deleteLibrary(name, function (err) {
+                if (err) {
+                    Logger.errlog.log("Failed to delete library for " + name + ": " + err);
+                }
+            });
+
+            module.exports.deleteAllRanks(name, function (err) {
+                if (err) {
+                    Logger.errlog.log("Failed to delete ranks for " + name + ": " + err);
+                }
+            });
+
+            fs.unlink(path.join(__dirname, "..", "..", "chandump", name),
+                      function (err) {
+                if (err && err.code !== "ENOENT") {
+                    Logger.errlog.log("Deleting chandump failed:");
+                    Logger.errlog.log(err);
+                }
+            });
+
+            callback(err, !Boolean(err));
         });
     },
 
@@ -313,8 +292,8 @@ module.exports = {
             return;
         }
 
-        db.query("SELECT name,rank FROM `chan_" + chan + "_ranks` WHERE name=?",
+        db.query("SELECT * FROM `channel_ranks` WHERE name=? AND channel=?",
-                 [name],
+                 [name, chan],
         function (err, rows) {
             if (err) {
                 callback(err, -1);
@@ -344,8 +323,12 @@ module.exports = {
         }
 
         var replace = "(" + names.map(function () { return "?"; }).join(",") + ")";
-        db.query("SELECT name,rank FROM `chan_" + chan + "_ranks` WHERE name IN " +
+
-                 replace, names,
+        /* Last substitution is the channel to select ranks for */
+        names.push(chan);
+
+        db.query("SELECT * FROM `channel_ranks` WHERE name IN " +
+                 replace + " AND channel=?", names,
         function (err, rows) {
             if (err) {
                 callback(err, []);
@@ -369,7 +352,7 @@ module.exports = {
             return;
         }
 
-        db.query("SELECT name,rank FROM `chan_" + chan + "_ranks` WHERE 1", callback);
+        db.query("SELECT * FROM `channel_ranks` WHERE channel=?", [chan], callback);
     },
 
     /**
@@ -390,25 +373,9 @@ module.exports = {
             return;
         }
 
-        db.query("INSERT INTO `chan_" + chan + "_ranks` (name, rank) VALUES (?, ?) " +
+        db.query("INSERT INTO `channel_ranks` VALUES (?, ?, ?) " +
-                 "ON DUPLICATE KEY UPDATE rank=?", [name, rank, rank], callback);
+                 "ON DUPLICATE KEY UPDATE rank=?",
-    },
+                 [name, rank, chan, rank, chan], callback);
-
-    /**
-     * Inserts a new user rank entry without clobbering an existing one
-     */
-    newRank: function (chan, name, rank, callback) {
-        if (typeof callback !== "function") {
-            callback = blackHole;
-        }
-
-        if (!valid(chan)) {
-            callback("Invalid channel name", null);
-            return;
-        }
-
-        db.query("INSERT INTO `chan_" + chan + "_ranks` (name, rank) VALUES (?, ?) " +
-                 "ON DUPLICATE KEY UPDATE rank=rank", [name, rank], callback);
     },
 
     /**
@@ -424,7 +391,24 @@ module.exports = {
             return;
         }
 
-        db.query("DELETE FROM `chan_" + chan + "_ranks` WHERE name=?", [name], callback);
+        db.query("DELETE FROM `channel_ranks` WHERE name=? AND channel=?", [name, chan],
+            callback);
+    },
+
+    /**
+     * Removes all ranks for a channel
+     */
+    deleteAllRanks: function (chan, callback) {
+        if (typeof callback !== "function") {
+            callback = blackHole;
+        }
+
+        if (!valid(chan)) {
+            callback("Invalid channel name", null);
+            return;
+        }
+
+        db.query("DELETE FROM `channel_ranks` WHERE channel=?", [chan], callback);
     },
 
     /**
@@ -445,9 +429,10 @@ module.exports = {
             codec: media.meta.codec
         });
 
-        db.query("INSERT INTO `chan_" + chan + "_library` (id, title, seconds, type, meta) " +
+        db.query("INSERT INTO `channel_libraries` " +
-                 "VALUES (?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE id=id",
+                 "(id, title, seconds, type, meta, channel) " +
-                 [media.id, media.title, media.seconds, media.type, meta], callback);
+                 "VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE id=id",
+                 [media.id, media.title, media.seconds, media.type, meta, chan], callback);
     },
 
     /**
@@ -463,7 +448,7 @@ module.exports = {
             return;
         }
 
-        db.query("SELECT * FROM `chan_" + chan + "_library` WHERE id=?", [id],
+        db.query("SELECT * FROM `channel_libraries` WHERE id=? AND channel=?", [id, chan],
         function (err, rows) {
             if (err) {
                 callback(err, null);
@@ -486,8 +471,8 @@ module.exports = {
             return;
         }
 
-        db.query("SELECT * FROM `chan_" + chan + "_library` WHERE title LIKE ?",
+        db.query("SELECT * FROM `channel_libraries` WHERE title LIKE ? AND channel=?",
-                 ["%" + search + "%"], callback);
+                 ["%" + search + "%", chan], callback);
     },
 
     /**
@@ -503,7 +488,24 @@ module.exports = {
             return;
         }
 
-        db.query("DELETE FROM `chan_" + chan + "_library` WHERE id=?", [id], callback);
+        db.query("DELETE FROM `channel_libraries` WHERE id=? AND channel=?",
+            [id, chan], callback);
+    },
+
+    /**
+     * Deletes all library entries for a channel
+     */
+    deleteLibrary: function (chan, callback) {
+        if (typeof callback !== "function") {
+            callback = blackHole;
+        }
+
+        if (!valid(chan)) {
+            callback("Invalid channel name", null);
+            return;
+        }
+
+        db.query("DELETE FROM `channel_libraries` WHERE channel=?", [chan], callback);
     },
 
     /**
@@ -519,8 +521,9 @@ module.exports = {
             return;
         }
 
-        db.query("INSERT INTO `chan_" + chan + "_bans` (ip, name, reason, bannedby) " +
+        db.query("INSERT INTO `channel_bans` (ip, name, reason, bannedby, channel) " +
-                 "VALUES (?, ?, ?, ?)", [ip, name, note, bannedby], callback);
+                 "VALUES (?, ?, ?, ?, ?)",
+            [ip, name, note, bannedby, chan], callback);
     },
 
     /**
@@ -539,8 +542,8 @@ module.exports = {
         var range = util.getIPRange(ip);
         var wrange = util.getWideIPRange(ip);
 
-        db.query("SELECT * FROM `chan_" + chan + "_bans` WHERE ip IN (?, ?, ?)",
+        db.query("SELECT * FROM `channel_bans` WHERE ip IN (?, ?, ?) AND channel=?",
-        [ip, range, wrange],
+        [ip, range, wrange, chan],
         function (err, rows) {
             callback(err, err ? false : rows.length > 0);
         });
@@ -559,7 +562,7 @@ module.exports = {
             return;
         }
 
-        db.query("SELECT * FROM `chan_" + chan + "_bans` WHERE name=?", [name],
+        db.query("SELECT * FROM `channel_bans` WHERE name=? AND channel=?", [name, chan],
         function (err, rows) {
             callback(err, err ? false : rows.length > 0);
         });
@@ -578,41 +581,7 @@ module.exports = {
             return;
         }
 
-        db.query("SELECT * FROM `chan_" + chan + "_bans` WHERE 1", callback);
+        db.query("SELECT * FROM `channel_bans` WHERE channel=?", [chan], callback);
-    },
-
-    /**
-     * Removes a ban from the banlist
-     */
-    unbanName: function (chan, name, callback) {
-        if (typeof callback !== "function") {
-            callback = blackHole;
-        }
-
-        if (!valid(chan)) {
-            callback("Invalid channel name", null);
-            return;
-        }
-
-        db.query("DELETE FROM `chan_" + chan + "_bans` WHERE ip='*' AND name=?",
-                 [name], callback);
-    },
-
-    /**
-     * Removes a ban from the banlist
-     */
-    unbanIP: function (chan, ip, callback) {
-        if (typeof callback !== "function") {
-            callback = blackHole;
-        }
-
-        if (!valid(chan)) {
-            callback("Invalid channel name", null);
-            return;
-        }
-
-        db.query("DELETE FROM `chan_" + chan + "_bans` WHERE ip=?",
-                 [ip], callback);
     },
 
     /**
@@ -628,7 +597,23 @@ module.exports = {
             return;
         }
 
-        db.query("DELETE FROM `chan_" + chan + "_bans` WHERE id=?",
+        db.query("DELETE FROM `channel_bans` WHERE id=? AND channel=?",
-                 [id], callback);
+                 [id, chan], callback);
+    },
+
+    /**
+     * Removes all bans from a channel
+     */
+    deleteBans: function (chan, id, callback) {
+        if (typeof callback !== "function") {
+            callback = blackHole;
+        }
+
+        if (!valid(chan)) {
+            callback("Invalid channel name", null);
+            return;
+        }
+
+        db.query("DELETE FROM `channel_bans` WHERE channel=?", [chan], callback);
     }
 };

lib/database/tables.js

@@ -72,10 +72,44 @@ const TBL_META = "" +
         "PRIMARY KEY (`key`))" +
     "CHARACTER SET utf8";
 
+const TBL_LIBRARIES = "" +
+    "CREATE TABLE IF NOT EXISTS `channel_libraries` (" +
+        "`id` VARCHAR(255) NOT NULL," +
+        "`title` VARCHAR(255) NOT NULL," +
+        "`seconds` INT NOT NULL," +
+        "`type` VARCHAR(2) NOT NULL," +
+        "`meta` TEXT NOT NULL," +
+        "`channel` VARCHAR(30) NOT NULL," +
+        "PRIMARY KEY(`id`, `channel`), INDEX(`channel`, `title`)" +
+    ") CHARACTER SET utf8";
+
+const TBL_RANKS = "" +
+    "CREATE TABLE IF NOT EXISTS `channel_ranks` (" +
+        "`name` VARCHAR(20) NOT NULL," +
+        "`rank` INT NOT NULL," +
+        "`channel` VARCHAR(30) NOT NULL," +
+        "PRIMARY KEY(`name`, `channel`)" +
+    ") CHARACTER SET utf8";
+
+const TBL_BANS = "" +
+    "CREATE TABLE IF NOT EXISTS `channel_bans` (" +
+        "`id` INT NOT NULL AUTO_INCREMENT," +
+        "`ip` VARCHAR(39) NOT NULL," +
+        "`name` VARCHAR(20) NOT NULL," +
+        "`bannedby` VARCHAR(20) NOT NULL," +
+        "`reason` VARCHAR(255) NOT NULL," +
+        "`channel` VARCHAR(30) NOT NULL," +
+    "PRIMARY KEY (`id`, `channel`), UNIQUE (`name`, `ip`, `channel`), " +
+    "INDEX (`ip`, `channel`), INDEX (`name`, `channel`)" +
+    ") CHARACTER SET utf8";
+
 module.exports.init = function (queryfn, cb) {
     var tables = {
         users: TBL_USERS,
         channels: TBL_CHANNELS,
+        channel_libraries: TBL_LIBRARIES,
+        channel_ranks: TBL_RANKS,
+        channel_bans: TBL_BANS,
         global_bans: TBL_GLOBAL_BANS,
         password_reset: TBL_PASSWORD_RESET,
         user_playlists: TBL_USER_PLAYLISTS,
@@ -104,45 +138,3 @@ module.exports.init = function (queryfn, cb) {
         cb(hasError);
     });
 };
-
-module.exports.createChannelTables = function (name, queryfn, cb) {
-    var createRanksTable = function () {
-        queryfn("CREATE TABLE `chan_" + name + "_ranks` (" +
-                    "`name` VARCHAR(20) NOT NULL," +
-                    "`rank` INT NOT NULL," +
-                "PRIMARY KEY (`name`)) " +
-                "CHARACTER SET utf8", createLibraryTable);
-    };
-
-    var createLibraryTable = function (err) {
-        if (err) {
-            cb(err);
-            return;
-        }
-        queryfn("CREATE TABLE `chan_" + name + "_library` (" +
-                    "`id` VARCHAR(255) NOT NULL," +
-                    "`title` VARCHAR(255) NOT NULL," +
-                    "`seconds` INT NOT NULL," +
-                    "`type` VARCHAR(2) NOT NULL," +
-                    "`meta` TEXT NOT NULL," +
-                "PRIMARY KEY (`id`))" +
-                "CHARACTER SET utf8", createBansTable);
-    };
-
-    var createBansTable = function (err) {
-        if (err) {
-            cb(err);
-            return;
-        }
-        queryfn("CREATE TABLE `chan_" + name + "_bans` (" +
-                    "`id` INT NOT NULL AUTO_INCREMENT," +
-                    "`ip` VARCHAR(39) NOT NULL," +
-                    "`name` VARCHAR(20) NOT NULL," +
-                    "`bannedby` VARCHAR(20) NOT NULL," +
-                    "`reason` VARCHAR(255) NOT NULL," +
-                "PRIMARY KEY (`id`), UNIQUE (`name`, `ip`))" +
-                "CHARACTER SET utf8", cb);
-    };
-
-    createRanksTable();
-};

lib/database/update.js

@@ -2,7 +2,7 @@ var db = require("../database");
 var Logger = require("../logger");
 var Q = require("q");
 
-const DB_VERSION = 3;
+const DB_VERSION = 4;
 var hasUpdates = [];
 
 module.exports.checkVersion = function () {
@@ -25,6 +25,7 @@ module.exports.checkVersion = function () {
             }
             var next = function () {
                 hasUpdates.push(v);
+                Logger.syslog.log("Updated database to version " + v);
                 if (v < DB_VERSION) {
                     update(v++, next);
                 } else {
@@ -40,6 +41,17 @@ module.exports.checkVersion = function () {
 function update(version, cb) {
     if (version < 3 && hasUpdates.indexOf(2) < 0) {
         addMetaColumnToLibraries(cb);
+    } else if (version < 4) {
+        Q.allSettled([
+            Q.nfcall(mergeChannelLibraries),
+            Q.nfcall(mergeChannelRanks),
+            Q.nfcall(mergeChannelBans)
+        ]).done(function () {
+            Logger.syslog.log("Merged channel tables.  Please verify that everything " +
+                              "is working correctly, and then type '/delete_old_tables'" +
+                              " into the CyTube process to remove the unused tables.");
+            cb();
+        })
     }
 }
 
@@ -68,3 +80,143 @@ function addMetaColumnToLibraries(cb) {
         Logger.errlog.log("Adding meta column to library tables failed: " + err);
     }).done(cb);
 }
+
+function mergeChannelLibraries(cb) {
+    Q.nfcall(db.query, "SHOW TABLES")
+    .then(function (rows) {
+        rows = rows.map(function (r) {
+            return r[Object.keys(r)[0]];
+        }).filter(function (r) {
+            return r.match(/chan_(.*)?_library$/);
+        });
+
+        var queue = [];
+        rows.forEach(function (table) {
+            var name = table.match(/chan_(.*?)_library$/)[1];
+            queue.push(Q.nfcall(db.query,
+                "INSERT INTO `channel_libraries` SELECT id, title, seconds, type, meta, ?" +
+                " AS channel FROM `" + table + "`", [name])
+                .then(function () {
+                    Logger.syslog.log("Copied " + table + " to channel_libraries");
+                }).catch(function (err) {
+                    Logger.errlog.log("Copying " + table + " to channel_libraries failed: " +
+                        err);
+                    if (err.stack) {
+                        Logger.errlog.log(err.stack);
+                    }
+                })
+            );
+        });
+
+        return Q.all(queue);
+    }).catch(function (err) {
+        Logger.errlog.log("Copying libraries to channel_libraries failed: " + err);
+        if (err.stack) {
+            Logger.errlog.log(err.stack);
+        }
+    }).done(function () { cb(null); });
+}
+
+function mergeChannelRanks(cb) {
+    Q.nfcall(db.query, "SHOW TABLES")
+    .then(function (rows) {
+        rows = rows.map(function (r) {
+            return r[Object.keys(r)[0]];
+        }).filter(function (r) {
+            return r.match(/chan_(.*?)_ranks$/);
+        });
+
+        var queue = [];
+        rows.forEach(function (table) {
+            var name = table.match(/chan_(.*?)_ranks$/)[1];
+            queue.push(Q.nfcall(db.query,
+                "INSERT INTO `channel_ranks` SELECT name, rank, ?" +
+                " AS channel FROM `" + table + "`", [name])
+                .then(function () {
+                    Logger.syslog.log("Copied " + table + " to channel_ranks");
+                }).catch(function (err) {
+                    Logger.errlog.log("Copying " + table + " to channel_ranks failed: " +
+                        err);
+                    if (err.stack) {
+                        Logger.errlog.log(err.stack);
+                    }
+                })
+            );
+        });
+
+        return Q.all(queue);
+    }).catch(function (err) {
+        Logger.errlog.log("Copying ranks to channel_ranks failed: " + err);
+        if (err.stack) {
+            Logger.errlog.log(err.stack);
+        }
+    }).done(function () { cb(null); });
+}
+
+function mergeChannelBans(cb) {
+    Q.nfcall(db.query, "SHOW TABLES")
+    .then(function (rows) {
+        rows = rows.map(function (r) {
+            return r[Object.keys(r)[0]];
+        }).filter(function (r) {
+            return r.match(/chan_(.*?)_bans$/);
+        });
+
+        var queue = [];
+        rows.forEach(function (table) {
+            var name = table.match(/chan_(.*?)_bans$/)[1];
+            queue.push(Q.nfcall(db.query,
+                "INSERT INTO `channel_bans` SELECT id, ip, name, bannedby, reason, ?" +
+                " AS channel FROM `" + table + "`", [name])
+                .then(function () {
+                    Logger.syslog.log("Copied " + table + " to channel_bans");
+                }).catch(function (err) {
+                    Logger.errlog.log("Copying " + table + " to channel_bans failed: " +
+                        err);
+                    if (err.stack) {
+                        Logger.errlog.log(err.stack);
+                    }
+                })
+            );
+        });
+
+        return Q.all(queue);
+    }).catch(function (err) {
+        Logger.errlog.log("Copying ranks to channel_bans failed: " + err);
+        if (err.stack) {
+            Logger.errlog.log(err.stack);
+        }
+    }).done(function () { cb(null); });
+}
+
+module.exports.deleteOldChannelTables = function (cb) {
+    Q.nfcall(db.query, "SHOW TABLES")
+    .then(function (rows) {
+        rows = rows.map(function (r) {
+            return r[Object.keys(r)[0]];
+        }).filter(function (r) {
+            return r.match(/chan_(.*?)_(library|ranks|bans)$/);
+        });
+
+        var queue = [];
+        rows.forEach(function (table) {
+            queue.push(Q.nfcall(db.query, "DROP TABLE `" + table + "`")
+                .then(function () {
+                    Logger.syslog.log("Deleted " + table);
+                }).catch(function (err) {
+                    Logger.errlog.log("Deleting " + table + " failed: " + err);
+                    if (err.stack) {
+                        Logger.errlog.log(err.stack);
+                    }
+                })
+            );
+        });
+
+        return Q.all(queue);
+    }).catch(function (err) {
+        Logger.errlog.log("Deleting old tables failed: " + err);
+        if (err.stack) {
+            Logger.errlog.log(err.stack);
+        }
+    }).done(cb);
+};

lib/server.js

@@ -9,7 +9,7 @@ The above copyright notice and this permission notice shall be included in all c
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
-const VERSION = "3.2.2";
+const VERSION = "3.3.0";
 var singleton = null;
 var Config = require("./config");
 

package.json

@@ -2,7 +2,7 @@
   "author": "Calvin Montgomery",
   "name": "CyTube",
   "description": "Online media synchronizer and chat",
-  "version": "3.2.2",
+  "version": "3.3.0",
   "repository": {
     "url": "http://github.com/calzoneman/sync"
   },