From 87b40b679a3820002f109282817ffdff8e19721e Mon Sep 17 00:00:00 2001 From: calzoneman Date: Sat, 8 Feb 2014 00:55:45 -0600 Subject: [PATCH] Refactor database tables init; make 2.x import script --- import.js | 493 +++++++++++++++++++++++++++++++++++++++ lib/database.js | 80 +------ lib/database/accounts.js | 14 -- lib/database/channels.js | 68 +----- lib/database/tables.js | 147 ++++++++++++ 5 files changed, 653 insertions(+), 149 deletions(-) create mode 100644 import.js create mode 100644 lib/database/tables.js diff --git a/import.js b/import.js new file mode 100644 index 00000000..f7842c64 --- /dev/null +++ b/import.js @@ -0,0 +1,493 @@ +/** + * Utility for importing a CyTube 2.4.6 database to 3.0 + */ + +var mysql = require("mysql"); +var AsyncQueue = require("./lib/asyncqueue"); +var tables = require("./lib/database/tables"); + +var olddb = { + host: "", + user: "", + password: "", + database: "" +}; + +var newdb = { + host: "", + user: "", + password: "", + database: "" +}; + +var oldpool; +var newpool; + +function query(pool, query, sub, callback) { + // 2nd argument is optional + if (typeof sub === "function") { + callback = sub; + sub = false; + } + + if (typeof callback !== "function") { + callback = function () { }; + } + + pool.getConnection(function (err, conn) { + if (err) { + console.log("[ERROR] DB connection failed: " + err); + callback("Database failure", null); + } else { + function cback(err, res) { + if (err) { + console.log("[ERROR] DB query failed: " + query); + if (sub) { + console.log("[ERROR] Substitutions: " + sub); + } + console.log("[ERROR] " + err); + callback("Database failure", null); + } else { + callback(null, res); + } + conn.release(); + } + + if (sub) { + conn.query(query, sub, cback); + } else { + conn.query(query, cback); + } + } + }); +}; + +var queryOld; +var queryNew; + +function chain(/* arguments */) { + var args = Array.prototype.slice.call(arguments); + var cb = args.pop(); + var next = function () { + if (args.length > 0) { + args.shift()(next); + } else { + cb(); + } + }; + + next(); +} + +/** + * Imports entries from the registrations table of 2.4.6 to the users table of 3.0 + */ +function importUsers(cb) { + console.log("[INFO] Importing users"); + var insert = "INSERT INTO `users` (`name`, `password`, `global_rank`, " + + "`email`, `profile`, `time`) VALUES (?, ?, ?, ?, ?, ?)"; + queryOld("SELECT * FROM `registrations`", function (err, rows) { + if (err) { + cb(err); + return; + } + + rows.sort(function (a, b) { + return a.id - b.id; + }); + + var aq = new AsyncQueue(); + rows.forEach(function (r) { + var data = [r.uname, r.pw, r.global_rank, r.email, + JSON.stringify({ image: r.profile_image, text: r.profile_text }), + Date.now()]; + aq.queue(function (lock) { + queryNew(insert, data, function (err) { + if (!err) { + console.log("Imported user " + r.uname); + } + lock.release(); + }); + }); + }); + + aq.queue(function (lock) { + lock.release(); + cb(); + }); + }); +} + +/** + * Imports channel registration entries from `channels` table + */ +function importChannelRegistrations(cb) { + var insert = "INSERT INTO `channels` (`name`, `owner`, `time`) VALUES (?, ?, ?)"; + + queryOld("SELECT * FROM channels", function (err, rows) { + if (err) { + cb(err); + return; + } + + rows.sort(function (a, b) { + return a.id - b.id; + }); + + var aq = new AsyncQueue(); + rows.forEach(function (r) { + var data = [r.name, r.owner, Date.now()]; + aq.queue(function (lock) { + queryNew(insert, data, function (err) { + if (!err) { + console.log("Imported channel record " + r.name + " (" + r.owner + ")"); + } + lock.release(); + }); + }); + }); + + aq.queue(function (lock) { + lock.release(); + cb(); + }); + }); +} + +/** + * Imports ranks/bans/library + */ +function importChannelTables(cb) { + console.log("Importing channel ranks, libraries, bans"); + queryOld("SELECT * FROM `channels`", function (err, rows) { + if (err) { + cb(err); + return; + } + + var aq = new AsyncQueue(); + + rows.forEach(function (r) { + aq.queue(function (lock) { + console.log("Creating channel tables for "+r.name); + tables.createChannelTables(r.name, queryNew, function () { + copyChannelTables(r.name, function () { + lock.release(); + }); + }); + }); + }); + + aq.queue(function (lock) { + lock.release(); + cb(); + }); + }); +} + +function copyChannelTables(name, cb) { + var copyRanks = function () { + queryOld("SELECT * FROM `chan_"+name+"_ranks`", function (err, rows) { + if (err) { + cb(err); + return; + } + + rows = rows.filter(function (r) { + return r.rank > 1; + }); + + rows = rows.map(function (r) { + if (r.rank === 10) { + r.rank = 4; + } else if (r.rank >= 3 && r.rank < 10) { + r.rank = 3; + } + return [r.name, r.rank]; + }); + + if (rows.length === 0) { + console.log("`chan_"+name+"_ranks` is empty"); + copyLibrary(); + return; + } + + console.log("Copying `chan_"+name+"_ranks`"); + queryNew("INSERT INTO `chan_"+name+"_ranks` VALUES ?", [rows], copyLibrary); + }); + }; + + var copyLibrary = function () { + queryOld("SELECT * FROM `chan_"+name+"_library`", function (err, rows) { + if (err) { + cb(err); + return; + } + + rows = rows.map(function (r) { + return [r.id, r.title, r.seconds, r.type]; + }); + + if (rows.length === 0) { + console.log("`chan_"+name+"_library` is empty"); + copyBans(); + return; + } + + var subs = []; + while (rows.length > 1000) { + subs.push(rows.slice(0, 1000)); + rows = rows.slice(1000); + } + + if (rows.length > 0) { + subs.push(rows); + } + + if (subs.length > 1) { + console.log("`chan_"+name+"_library` is >1000 rows, requires multiple inserts"); + } + + var aq = new AsyncQueue(); + subs.forEach(function (s) { + aq.queue(function (lock) { + console.log("Copying `chan_"+name+"_library`"); + queryNew("INSERT INTO `chan_"+name+"_library` VALUES ?", + [s], function () { + lock.release(); + }); + }); + }); + + aq.queue(function (lock) { + lock.release(); + copyBans(); + }); + }); + }; + + var copyBans = function () { + queryOld("SELECT * FROM `chan_"+name+"_bans`", function (err, rows) { + if (err) { + cb(err); + return; + } + + rows = rows.map(function (r) { + return [r.id, r.ip, r.name, r.bannedby, r.reason]; + }); + + if (rows.length === 0) { + console.log("`chan_"+name+"_bans` is empty"); + cb(); + return; + } + + console.log("Copying `chan_"+name+"_bans`"); + queryNew("INSERT INTO `chan_"+name+"_bans` VALUES ?", [rows], cb); + }); + }; + + copyRanks(); +} + +function importGlobalBans(cb) { + console.log("Importing global bans"); + queryOld("SELECT * FROM `global_bans`", function (err, bans) { + if (err) { + cb(err); + return; + } + + bans = bans.map(function (b) { + return [b.ip, b.reason]; + }); + queryNew("INSERT INTO `global_bans` VALUES ?", [bans], cb); + }); +} + +function importUserPlaylists(cb) { + console.log("Importing user playlists"); + queryOld("SELECT * FROM `user_playlists`", function (err, pls) { + if (err) { + cb(err); + return; + } + + pls = pls.map(function (pl) { + return [pl.user, pl.name, pl.contents, pl.count, pl.duration]; + }); + var subs = []; + while (pls.length > 10) { + subs.push(pls.slice(0, 10)); + pls = pls.slice(10); + } + + if (pls.length > 0) { + subs.push(pls); + } + + var aq = new AsyncQueue(); + subs.forEach(function (s) { + aq.queue(function (lock) { + queryNew("INSERT INTO `user_playlists` VALUES ?", [s], function () { + lock.release(); + }); + }); + }); + + aq.queue(function (lock) { + lock.release(); + cb(); + }); + }); +} + +function importAliases(cb) { + console.log("Importing aliases"); + queryOld("SELECT * FROM `aliases`", function (err, aliases) { + if (err) { + cb(err); + return; + } + + aliases = aliases.map(function (al) { + return [al.visit_id, al.ip, al.name, al.time]; + }); + + var subs = []; + while (aliases.length > 1000) { + subs.push(aliases.slice(0, 1000)); + aliases = aliases.slice(1000); + } + + if (aliases.length > 0) { + subs.push(aliases); + } + + var aq = new AsyncQueue(); + subs.forEach(function (s) { + aq.queue(function (lock) { + queryNew("INSERT INTO `aliases` VALUES ?", [s], function () { + lock.release(); + }); + }); + }); + + aq.queue(function (lock) { + lock.release(); + cb(); + }); + }); +} + +function main() { + var aq = new AsyncQueue(); + var readline = require("readline"); + console.log("This script will generate a lot of text output, both informational and " + + "possibly errors. I recommend running it as `node import.js | " + + "tee import.log` or similar to pipe output to a log file for easy reading"); + var rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); + aq.queue(function (lock) { + rl.question("2.x host: ", function (host) { + olddb.host = host; + lock.release(); + }); + }); + aq.queue(function (lock) { + rl.question("2.x username: ", function (user) { + olddb.user = user; + lock.release(); + }); + }); + aq.queue(function (lock) { + rl.question("2.x password: ", function (pw) { + olddb.password = pw; + lock.release(); + }); + }); + aq.queue(function (lock) { + rl.question("2.x database: ", function (db) { + olddb.database = db; + lock.release(); + }); + }); + aq.queue(function (lock) { + rl.question("3.0 host: ", function (host) { + newdb.host = host; + lock.release(); + }); + }); + aq.queue(function (lock) { + rl.question("3.0 username: ", function (user) { + newdb.user = user; + lock.release(); + }); + }); + aq.queue(function (lock) { + rl.question("3.0 password: ", function (pw) { + newdb.password = pw; + lock.release(); + }); + }); + aq.queue(function (lock) { + rl.question("3.0 database: ", function (db) { + newdb.database = db; + lock.release(); + }); + }); + aq.queue(function (lock) { + oldpool = mysql.createPool(olddb); + newpool = mysql.createPool(newdb); + queryOld = query.bind(this, oldpool); + queryNew = query.bind(this, newpool); + startImport(); + }); +} + +function startImport() { + tables.init(queryNew, function (err) { + if (!err) { + var aq = new AsyncQueue(); + aq.queue(function (lock) { + importUsers(function () { + lock.release(); + }); + }); + aq.queue(function (lock) { + importChannelRegistrations(function () { + lock.release(); }); + }); + aq.queue(function (lock) { + importChannelTables(function () { + lock.release(); + }); + }); + aq.queue(function (lock) { + importGlobalBans(function () { + lock.release(); + }); + }); + aq.queue(function (lock) { + importUserPlaylists(function () { + lock.release(); + }); + }); + aq.queue(function (lock) { + importAliases(function () { + lock.release(); + }); + }); + aq.queue(function (lock) { + process.exit(0); + }); + } else { + console.log("[ERROR] Aborting due to errors initializing tables"); + } + }); +} + +main(); diff --git a/lib/database.js b/lib/database.js index 0b0872d1..d4f2e0bb 100644 --- a/lib/database.js +++ b/lib/database.js @@ -4,6 +4,7 @@ var $util = require("./utilities"); var Logger = require("./logger"); var Config = require("./config"); var Server = require("./server"); +var tables = require("./database/tables"); var pool = null; var global_ipbans = {}; @@ -23,7 +24,13 @@ module.exports.init = function () { Logger.errlog.log("! DB connection failed"); return; } else { - module.exports.initGlobalTables(); + tables.init(module.exports.query, function (err) { + if (err) { + return; + } + require("./database/update").checkVersion(); + module.exports.loadAnnouncement(); + }); // Refresh global IP bans module.exports.listGlobalBans(); } @@ -31,9 +38,7 @@ module.exports.init = function () { global_ipbans = {}; module.exports.users = require("./database/accounts"); - module.exports.users.init(); module.exports.channels = require("./database/channels"); - module.exports.channels.init(); }; /** @@ -85,75 +90,6 @@ function blackHole() { } -module.exports.initGlobalTables = function () { - var fail = function (table) { - return function (err) { - if (err) { - Logger.errlog.log("Failed to initialize " + table); - } - }; - }; - var query = module.exports.query; - query("CREATE TABLE IF NOT EXISTS `global_bans` (" + - "`ip` VARCHAR(39) NOT NULL," + - "`reason` VARCHAR(255) NOT NULL," + - "PRIMARY KEY (`ip`)) " + - "CHARACTER SET utf8", - fail("global_bans")); - - query("CREATE TABLE IF NOT EXISTS `password_reset` (" + - "`ip` VARCHAR(39) NOT NULL," + - "`name` VARCHAR(20) NOT NULL," + - "`hash` VARCHAR(64) NOT NULL," + - "`email` VARCHAR(255) NOT NULL," + - "`expire` BIGINT NOT NULL," + - "PRIMARY KEY (`name`))" + - "CHARACTER SET utf8", - fail("password_reset")); - - query("CREATE TABLE IF NOT EXISTS `user_playlists` (" + - "`user` VARCHAR(20) NOT NULL," + - "`name` VARCHAR(255) NOT NULL," + - "`contents` MEDIUMTEXT NOT NULL," + - "`count` INT NOT NULL," + - "`duration` INT NOT NULL," + - "PRIMARY KEY (`user`, `name`))" + - "CHARACTER SET utf8", - fail("user_playlists")); - - query("CREATE TABLE IF NOT EXISTS `aliases` (" + - "`visit_id` INT NOT NULL AUTO_INCREMENT," + - "`ip` VARCHAR(39) NOT NULL," + - "`name` VARCHAR(20) NOT NULL," + - "`time` BIGINT NOT NULL," + - "PRIMARY KEY (`visit_id`), INDEX (`ip`))", - fail("aliases")); - - query("CREATE TABLE IF NOT EXISTS `stats` (" + - "`time` BIGINT NOT NULL," + - "`usercount` INT NOT NULL," + - "`chancount` INT NOT NULL," + - "`mem` INT NOT NULL," + - "PRIMARY KEY (`time`))" + - "CHARACTER SET utf8", - fail("stats")); - - query("CREATE TABLE IF NOT EXISTS `meta` (" + - "`key` VARCHAR(255) NOT NULL," + - "`value` TEXT NOT NULL," + - "PRIMARY KEY (`key`))" + - "CHARACTER SET utf8", - function (err, res) { - if (err) { - fail("meta")(err); - return; - } - - require("./database/update").checkVersion(); - module.exports.loadAnnouncement(); - }); -}; - /* REGION global bans */ /** diff --git a/lib/database/accounts.js b/lib/database/accounts.js index 3d4c9c2f..2672b1ff 100644 --- a/lib/database/accounts.js +++ b/lib/database/accounts.js @@ -7,21 +7,7 @@ var registrationLock = {}; var blackHole = function () { }; module.exports = { - /** - * Initialize the accounts table - */ init: function () { - db.query("CREATE TABLE IF NOT EXISTS `users` (" + - "`id` INT NOT NULL AUTO_INCREMENT," + - "`name` VARCHAR(20) NOT NULL," + - "`password` VARCHAR(64) NOT NULL," + - "`global_rank` INT NOT NULL," + - "`email` VARCHAR(255) NOT NULL," + - "`profile` TEXT NOT NULL," + - "`ip` VARCHAR(39) NOT NULL," + - "`time` BIGINT NOT NULL, " + - "PRIMARY KEY(`id`), INDEX(`name`)) " + - "CHARACTER SET utf8"); }, /** diff --git a/lib/database/channels.js b/lib/database/channels.js index 477bb7c0..dfb8254f 100644 --- a/lib/database/channels.js +++ b/lib/database/channels.js @@ -3,6 +3,7 @@ var valid = require("../utilities").isValidChannelName; var fs = require("fs"); var path = require("path"); var Logger = require("../logger"); +var tables = require("./tables"); var blackHole = function () { }; @@ -10,47 +11,13 @@ function dropTable(name, callback) { db.query("DROP TABLE `" + name + "`", callback); } -function createRanksTable(name, callback) { - db.query("CREATE TABLE `chan_" + name + "_ranks` (" + - "`name` VARCHAR(20) NOT NULL," + - "`rank` INT NOT NULL," + - "PRIMARY KEY (`name`)) " + - "CHARACTER SET utf8", callback); -} - -function createLibraryTable(name, callback) { - db.query("CREATE TABLE `chan_" + name + "_library` (" + - "`id` VARCHAR(255) NOT NULL," + - "`title` VARCHAR(255) NOT NULL," + - "`seconds` INT NOT NULL," + - "`type` VARCHAR(2) NOT NULL," + - "PRIMARY KEY (`id`))" + - "CHARACTER SET utf8", callback); -} - -function createBansTable(name, callback) { - db.query("CREATE TABLE `chan_" + name + "_bans` (" + - "`id` INT NOT NULL AUTO_INCREMENT," + - "`ip` VARCHAR(39) NOT NULL," + - "`name` VARCHAR(20) NOT NULL," + - "`bannedby` VARCHAR(20) NOT NULL," + - "`reason` VARCHAR(255) NOT NULL," + - "PRIMARY KEY (`id`), UNIQUE (`name`, `ip`))" + - "CHARACTER SET utf8", callback); -} - function initTables(name, owner, callback) { if (!valid(name)) { callback("Invalid channel name", null); return; } - createRanksTable(name, function (err) { - if (err) { - callback(err, null); - return; - } - + tables.createChannelTables(name, db.query.bind(db), function (err) { db.users.getGlobalRank(owner, function (err, rank) { if (err) { callback(err, null); @@ -62,45 +29,20 @@ function initTables(name, owner, callback) { module.exports.setRank(name, owner, rank, function (err) { if (err) { dropTable("chan_" + name + "_ranks"); + dropTable("chan_" + name + "_ranks"); + dropTable("chan_" + name + "_library"); callback(err, null); return; } - createLibraryTable(name, function (err) { - if (err) { - dropTable("chan_" + name + "_ranks"); - callback(err, null); - return; - } - - createBansTable(name, function (err) { - if (err) { - dropTable("chan_" + name + "_ranks"); - dropTable("chan_" + name + "_library"); - callback(err, null); - return; - } - - callback(null, true); - }); - }); + callback(null, true); }); }); }); } module.exports = { - /** - * Initialize the channels table - */ init: function () { - db.query("CREATE TABLE IF NOT EXISTS `channels` (" + - "`id` INT NOT NULL AUTO_INCREMENT," + - "`name` VARCHAR(30) NOT NULL," + - "`owner` VARCHAR(20) NOT NULL," + - "`time` BIGINT NOT NULL," + - "PRIMARY KEY (`id`), INDEX(`name`), INDEX(`owner`))" + - "CHARACTER SET utf8"); }, /** diff --git a/lib/database/tables.js b/lib/database/tables.js new file mode 100644 index 00000000..759bcc2d --- /dev/null +++ b/lib/database/tables.js @@ -0,0 +1,147 @@ +const TBL_USERS = "" + + "CREATE TABLE IF NOT EXISTS `users` (" + + "`id` INT NOT NULL AUTO_INCREMENT," + + "`name` VARCHAR(20) NOT NULL," + + "`password` VARCHAR(64) NOT NULL," + + "`global_rank` INT NOT NULL," + + "`email` VARCHAR(255) NOT NULL," + + "`profile` TEXT NOT NULL," + + "`ip` VARCHAR(39) NOT NULL," + "`time` BIGINT NOT NULL," + + "PRIMARY KEY(`id`)," + + "UNIQUE(`name`)) " + + "CHARACTER SET utf8"; + +const TBL_CHANNELS = "" + + "CREATE TABLE IF NOT EXISTS `channels` (" + + "`id` INT NOT NULL AUTO_INCREMENT," + + "`name` VARCHAR(30) NOT NULL," + + "`owner` VARCHAR(20) NOT NULL," + + "`time` BIGINT NOT NULL," + + "PRIMARY KEY (`id`), UNIQUE(`name`), INDEX(`owner`))" + + "CHARACTER SET utf8"; + +const TBL_GLOBAL_BANS = "" + + "CREATE TABLE IF NOT EXISTS `global_bans` (" + + "`ip` VARCHAR(39) NOT NULL," + + "`reason` VARCHAR(255) NOT NULL," + + "PRIMARY KEY (`ip`)) " + + "CHARACTER SET utf8"; + +const TBL_PASSWORD_RESET = "" + + "CREATE TABLE IF NOT EXISTS `password_reset` (" + + "`ip` VARCHAR(39) NOT NULL," + + "`name` VARCHAR(20) NOT NULL," + + "`hash` VARCHAR(64) NOT NULL," + + "`email` VARCHAR(255) NOT NULL," + + "`expire` BIGINT NOT NULL," + + "PRIMARY KEY (`name`))" + + "CHARACTER SET utf8"; + +const TBL_USER_PLAYLISTS = "" + + "CREATE TABLE IF NOT EXISTS `user_playlists` (" + + "`user` VARCHAR(20) NOT NULL," + + "`name` VARCHAR(255) NOT NULL," + + "`contents` MEDIUMTEXT NOT NULL," + + "`count` INT NOT NULL," + + "`duration` INT NOT NULL," + + "PRIMARY KEY (`user`, `name`))" + + "CHARACTER SET utf8"; + +const TBL_ALIASES = "" + + "CREATE TABLE IF NOT EXISTS `aliases` (" + + "`visit_id` INT NOT NULL AUTO_INCREMENT," + + "`ip` VARCHAR(39) NOT NULL," + + "`name` VARCHAR(20) NOT NULL," + + "`time` BIGINT NOT NULL," + + "PRIMARY KEY (`visit_id`), INDEX (`ip`)" + + ")"; + +const TBL_STATS = "" + + "CREATE TABLE IF NOT EXISTS `stats` (" + + "`time` BIGINT NOT NULL," + + "`usercount` INT NOT NULL," + + "`chancount` INT NOT NULL," + + "`mem` INT NOT NULL," + + "PRIMARY KEY (`time`))" + + "CHARACTER SET utf8"; + +const TBL_META = "" + + "CREATE TABLE IF NOT EXISTS `meta` (" + + "`key` VARCHAR(255) NOT NULL," + + "`value` TEXT NOT NULL," + + "PRIMARY KEY (`key`))" + + "CHARACTER SET utf8"; + +module.exports.init = function (queryfn, cb) { + var tables = { + users: TBL_USERS, + channels: TBL_CHANNELS, + global_bans: TBL_GLOBAL_BANS, + password_reset: TBL_PASSWORD_RESET, + user_playlists: TBL_USER_PLAYLISTS, + aliases: TBL_ALIASES, + stats: TBL_STATS, + meta: TBL_META + }; + + var AsyncQueue = require("../asyncqueue"); + var aq = new AsyncQueue(); + var hasError = false; + Object.keys(tables).forEach(function (tbl) { + aq.queue(function (lock) { + queryfn(tables[tbl], function (err) { + if (err) { + console.log(err); + hasError = true; + } + lock.release(); + }); + }); + }); + + aq.queue(function (lock) { + lock.release(); + cb(hasError); + }); +}; + +module.exports.createChannelTables = function (name, queryfn, cb) { + var createRanksTable = function () { + queryfn("CREATE TABLE `chan_" + name + "_ranks` (" + + "`name` VARCHAR(20) NOT NULL," + + "`rank` INT NOT NULL," + + "PRIMARY KEY (`name`)) " + + "CHARACTER SET utf8", createLibraryTable); + }; + + var createLibraryTable = function (err) { + if (err) { + cb(err); + return; + } + queryfn("CREATE TABLE `chan_" + name + "_library` (" + + "`id` VARCHAR(255) NOT NULL," + + "`title` VARCHAR(255) NOT NULL," + + "`seconds` INT NOT NULL," + + "`type` VARCHAR(2) NOT NULL," + + "PRIMARY KEY (`id`))" + + "CHARACTER SET utf8", createBansTable); + }; + + var createBansTable = function (err) { + if (err) { + cb(err); + return; + } + queryfn("CREATE TABLE `chan_" + name + "_bans` (" + + "`id` INT NOT NULL AUTO_INCREMENT," + + "`ip` VARCHAR(39) NOT NULL," + + "`name` VARCHAR(20) NOT NULL," + + "`bannedby` VARCHAR(20) NOT NULL," + + "`reason` VARCHAR(255) NOT NULL," + + "PRIMARY KEY (`id`), UNIQUE (`name`, `ip`))" + + "CHARACTER SET utf8", cb); + }; + + createRanksTable(); +};