moon
/
sync
mirror of https://github.com/calzoneman/sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor database tables init; make 2.x import script

This commit is contained in:
calzoneman 2014-02-08 00:55:45 -06:00
parent 0a480515d7
commit 87b40b679a
5 changed files with 653 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

493
import.js Normal file
View File

@ -0,0 +1,493 @@
/**
* Utility for importing a CyTube 2.4.6 database to 3.0
*/
var mysql = require("mysql");
var AsyncQueue = require("./lib/asyncqueue");
var tables = require("./lib/database/tables");
var olddb = {
host: "",
user: "",
password: "",
database: ""
};
var newdb = {
host: "",
user: "",
password: "",
database: ""
};
var oldpool;
var newpool;
function query(pool, query, sub, callback) {
// 2nd argument is optional
if (typeof sub === "function") {
callback = sub;
sub = false;
}
if (typeof callback !== "function") {
callback = function () { };
}
pool.getConnection(function (err, conn) {
if (err) {
console.log("[ERROR] DB connection failed: " + err);
callback("Database failure", null);
} else {
function cback(err, res) {
if (err) {
console.log("[ERROR] DB query failed: " + query);
if (sub) {
console.log("[ERROR] Substitutions: " + sub);
}
console.log("[ERROR] " + err);
callback("Database failure", null);
} else {
callback(null, res);
}
conn.release();
}
if (sub) {
conn.query(query, sub, cback);
} else {
conn.query(query, cback);
}
}
});
};
var queryOld;
var queryNew;
function chain(/* arguments */) {
var args = Array.prototype.slice.call(arguments);
var cb = args.pop();
var next = function () {
if (args.length > 0) {
args.shift()(next);
} else {
cb();
}
};
next();
}
/**
* Imports entries from the registrations table of 2.4.6 to the users table of 3.0
*/
function importUsers(cb) {
console.log("[INFO] Importing users");
var insert = "INSERT INTO `users` (`name`, `password`, `global_rank`, " +
"`email`, `profile`, `time`) VALUES (?, ?, ?, ?, ?, ?)";
queryOld("SELECT * FROM `registrations`", function (err, rows) {
if (err) {
cb(err);
return;
}
rows.sort(function (a, b) {
return a.id - b.id;
});
var aq = new AsyncQueue();
rows.forEach(function (r) {
var data = [r.uname, r.pw, r.global_rank, r.email,
JSON.stringify({ image: r.profile_image, text: r.profile_text }),
Date.now()];
aq.queue(function (lock) {
queryNew(insert, data, function (err) {
if (!err) {
console.log("Imported user " + r.uname);
}
lock.release();
});
});
});
aq.queue(function (lock) {
lock.release();
cb();
});
});
}
/**
* Imports channel registration entries from `channels` table
*/
function importChannelRegistrations(cb) {
var insert = "INSERT INTO `channels` (`name`, `owner`, `time`) VALUES (?, ?, ?)";
queryOld("SELECT * FROM channels", function (err, rows) {
if (err) {
cb(err);
return;
}
rows.sort(function (a, b) {
return a.id - b.id;
});
var aq = new AsyncQueue();
rows.forEach(function (r) {
var data = [r.name, r.owner, Date.now()];
aq.queue(function (lock) {
queryNew(insert, data, function (err) {
if (!err) {
console.log("Imported channel record " + r.name + " (" + r.owner + ")");
}
lock.release();
});
});
});
aq.queue(function (lock) {
lock.release();
cb();
});
});
}
/**
* Imports ranks/bans/library
*/
function importChannelTables(cb) {
console.log("Importing channel ranks, libraries, bans");
queryOld("SELECT * FROM `channels`", function (err, rows) {
if (err) {
cb(err);
return;
}
var aq = new AsyncQueue();
rows.forEach(function (r) {
aq.queue(function (lock) {
console.log("Creating channel tables for "+r.name);
tables.createChannelTables(r.name, queryNew, function () {
copyChannelTables(r.name, function () {
lock.release();
});
});
});
});
aq.queue(function (lock) {
lock.release();
cb();
});
});
}
function copyChannelTables(name, cb) {
var copyRanks = function () {
queryOld("SELECT * FROM `chan_"+name+"_ranks`", function (err, rows) {
if (err) {
cb(err);
return;
}
rows = rows.filter(function (r) {
return r.rank > 1;
});
rows = rows.map(function (r) {
if (r.rank === 10) {
r.rank = 4;
} else if (r.rank >= 3 && r.rank < 10) {
r.rank = 3;
}
return [r.name, r.rank];
});
if (rows.length === 0) {
console.log("`chan_"+name+"_ranks` is empty");
copyLibrary();
return;
}
console.log("Copying `chan_"+name+"_ranks`");
queryNew("INSERT INTO `chan_"+name+"_ranks` VALUES ?", [rows], copyLibrary);
});
};
var copyLibrary = function () {
queryOld("SELECT * FROM `chan_"+name+"_library`", function (err, rows) {
if (err) {
cb(err);
return;
}
rows = rows.map(function (r) {
return [r.id, r.title, r.seconds, r.type];
});
if (rows.length === 0) {
console.log("`chan_"+name+"_library` is empty");
copyBans();
return;
}
var subs = [];
while (rows.length > 1000) {
subs.push(rows.slice(0, 1000));
rows = rows.slice(1000);
}
if (rows.length > 0) {
subs.push(rows);
}
if (subs.length > 1) {
console.log("`chan_"+name+"_library` is >1000 rows, requires multiple inserts");
}
var aq = new AsyncQueue();
subs.forEach(function (s) {
aq.queue(function (lock) {
console.log("Copying `chan_"+name+"_library`");
queryNew("INSERT INTO `chan_"+name+"_library` VALUES ?",
[s], function () {
lock.release();
});
});
});
aq.queue(function (lock) {
lock.release();
copyBans();
});
});
};
var copyBans = function () {
queryOld("SELECT * FROM `chan_"+name+"_bans`", function (err, rows) {
if (err) {
cb(err);
return;
}
rows = rows.map(function (r) {
return [r.id, r.ip, r.name, r.bannedby, r.reason];
});
if (rows.length === 0) {
console.log("`chan_"+name+"_bans` is empty");
cb();
return;
}
console.log("Copying `chan_"+name+"_bans`");
queryNew("INSERT INTO `chan_"+name+"_bans` VALUES ?", [rows], cb);
});
};
copyRanks();
}
function importGlobalBans(cb) {
console.log("Importing global bans");
queryOld("SELECT * FROM `global_bans`", function (err, bans) {
if (err) {
cb(err);
return;
}
bans = bans.map(function (b) {
return [b.ip, b.reason];
});
queryNew("INSERT INTO `global_bans` VALUES ?", [bans], cb);
});
}
function importUserPlaylists(cb) {
console.log("Importing user playlists");
queryOld("SELECT * FROM `user_playlists`", function (err, pls) {
if (err) {
cb(err);
return;
}
pls = pls.map(function (pl) {
return [pl.user, pl.name, pl.contents, pl.count, pl.duration];
});
var subs = [];
while (pls.length > 10) {
subs.push(pls.slice(0, 10));
pls = pls.slice(10);
}
if (pls.length > 0) {
subs.push(pls);
}
var aq = new AsyncQueue();
subs.forEach(function (s) {
aq.queue(function (lock) {
queryNew("INSERT INTO `user_playlists` VALUES ?", [s], function () {
lock.release();
});
});
});
aq.queue(function (lock) {
lock.release();
cb();
});
});
}
function importAliases(cb) {
console.log("Importing aliases");
queryOld("SELECT * FROM `aliases`", function (err, aliases) {
if (err) {
cb(err);
return;
}
aliases = aliases.map(function (al) {
return [al.visit_id, al.ip, al.name, al.time];
});
var subs = [];
while (aliases.length > 1000) {
subs.push(aliases.slice(0, 1000));
aliases = aliases.slice(1000);
}
if (aliases.length > 0) {
subs.push(aliases);
}
var aq = new AsyncQueue();
subs.forEach(function (s) {
aq.queue(function (lock) {
queryNew("INSERT INTO `aliases` VALUES ?", [s], function () {
lock.release();
});
});
});
aq.queue(function (lock) {
lock.release();
cb();
});
});
}
function main() {
var aq = new AsyncQueue();
var readline = require("readline");
console.log("This script will generate a lot of text output, both informational and " +
"possibly errors. I recommend running it as `node import.js | " +
"tee import.log` or similar to pipe output to a log file for easy reading");
var rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
aq.queue(function (lock) {
rl.question("2.x host: ", function (host) {
olddb.host = host;
lock.release();
});
});
aq.queue(function (lock) {
rl.question("2.x username: ", function (user) {
olddb.user = user;
lock.release();
});
});
aq.queue(function (lock) {
rl.question("2.x password: ", function (pw) {
olddb.password = pw;
lock.release();
});
});
aq.queue(function (lock) {
rl.question("2.x database: ", function (db) {
olddb.database = db;
lock.release();
});
});
aq.queue(function (lock) {
rl.question("3.0 host: ", function (host) {
newdb.host = host;
lock.release();
});
});
aq.queue(function (lock) {
rl.question("3.0 username: ", function (user) {
newdb.user = user;
lock.release();
});
});
aq.queue(function (lock) {
rl.question("3.0 password: ", function (pw) {
newdb.password = pw;
lock.release();
});
});
aq.queue(function (lock) {
rl.question("3.0 database: ", function (db) {
newdb.database = db;
lock.release();
});
});
aq.queue(function (lock) {
oldpool = mysql.createPool(olddb);
newpool = mysql.createPool(newdb);
queryOld = query.bind(this, oldpool);
queryNew = query.bind(this, newpool);
startImport();
});
}
function startImport() {
tables.init(queryNew, function (err) {
if (!err) {
var aq = new AsyncQueue();
aq.queue(function (lock) {
importUsers(function () {
lock.release();
});
});
aq.queue(function (lock) {
importChannelRegistrations(function () {
lock.release(); });
});
aq.queue(function (lock) {
importChannelTables(function () {
lock.release();
});
});
aq.queue(function (lock) {
importGlobalBans(function () {
lock.release();
});
});
aq.queue(function (lock) {
importUserPlaylists(function () {
lock.release();
});
});
aq.queue(function (lock) {
importAliases(function () {
lock.release();
});
});
aq.queue(function (lock) {
process.exit(0);
});
} else {
console.log("[ERROR] Aborting due to errors initializing tables");
}
});
}
main();

80
lib/database.js
View File

@ -4,6 +4,7 @@ var $util = require("./utilities");
var Logger = require("./logger");
var Config = require("./config");
var Server = require("./server");
var tables = require("./database/tables");
var pool = null;
var global_ipbans = {};
@ -23,7 +24,13 @@ module.exports.init = function () {
Logger.errlog.log("! DB connection failed");
return;
} else {
module.exports.initGlobalTables();
tables.init(module.exports.query, function (err) {
if (err) {
return;
}
require("./database/update").checkVersion();
module.exports.loadAnnouncement();
});
// Refresh global IP bans
module.exports.listGlobalBans();
}
@ -31,9 +38,7 @@ module.exports.init = function () {
global_ipbans = {};
module.exports.users = require("./database/accounts");
module.exports.users.init();
module.exports.channels = require("./database/channels");
module.exports.channels.init();
};
/**
@ -85,75 +90,6 @@ function blackHole() {
}
module.exports.initGlobalTables = function () {
var fail = function (table) {
return function (err) {
if (err) {
Logger.errlog.log("Failed to initialize " + table);
}
};
};
var query = module.exports.query;
query("CREATE TABLE IF NOT EXISTS `global_bans` (" +
"`ip` VARCHAR(39) NOT NULL," +
"`reason` VARCHAR(255) NOT NULL," +
"PRIMARY KEY (`ip`)) " +
"CHARACTER SET utf8",
fail("global_bans"));
query("CREATE TABLE IF NOT EXISTS `password_reset` (" +
"`ip` VARCHAR(39) NOT NULL," +
"`name` VARCHAR(20) NOT NULL," +
"`hash` VARCHAR(64) NOT NULL," +
"`email` VARCHAR(255) NOT NULL," +
"`expire` BIGINT NOT NULL," +
"PRIMARY KEY (`name`))" +
"CHARACTER SET utf8",
fail("password_reset"));
query("CREATE TABLE IF NOT EXISTS `user_playlists` (" +
"`user` VARCHAR(20) NOT NULL," +
"`name` VARCHAR(255) NOT NULL," +
"`contents` MEDIUMTEXT NOT NULL," +
"`count` INT NOT NULL," +
"`duration` INT NOT NULL," +
"PRIMARY KEY (`user`, `name`))" +
"CHARACTER SET utf8",
fail("user_playlists"));
query("CREATE TABLE IF NOT EXISTS `aliases` (" +
"`visit_id` INT NOT NULL AUTO_INCREMENT," +
"`ip` VARCHAR(39) NOT NULL," +
"`name` VARCHAR(20) NOT NULL," +
"`time` BIGINT NOT NULL," +
"PRIMARY KEY (`visit_id`), INDEX (`ip`))",
fail("aliases"));
query("CREATE TABLE IF NOT EXISTS `stats` (" +
"`time` BIGINT NOT NULL," +
"`usercount` INT NOT NULL," +
"`chancount` INT NOT NULL," +
"`mem` INT NOT NULL," +
"PRIMARY KEY (`time`))" +
"CHARACTER SET utf8",
fail("stats"));
query("CREATE TABLE IF NOT EXISTS `meta` (" +
"`key` VARCHAR(255) NOT NULL," +
"`value` TEXT NOT NULL," +
"PRIMARY KEY (`key`))" +
"CHARACTER SET utf8",
function (err, res) {
if (err) {
fail("meta")(err);
return;
}
require("./database/update").checkVersion();
module.exports.loadAnnouncement();
});
};
/* REGION global bans */
/**

14
lib/database/accounts.js
View File

@ -7,21 +7,7 @@ var registrationLock = {};
var blackHole = function () { };
module.exports = {
/**
* Initialize the accounts table
*/
init: function () {
db.query("CREATE TABLE IF NOT EXISTS `users` (" +
"`id` INT NOT NULL AUTO_INCREMENT," +
"`name` VARCHAR(20) NOT NULL," +
"`password` VARCHAR(64) NOT NULL," +
"`global_rank` INT NOT NULL," +
"`email` VARCHAR(255) NOT NULL," +
"`profile` TEXT NOT NULL," +
"`ip` VARCHAR(39) NOT NULL," +
"`time` BIGINT NOT NULL, " +
"PRIMARY KEY(`id`), INDEX(`name`)) " +
"CHARACTER SET utf8");
},
/**

62
lib/database/channels.js
View File

@ -3,6 +3,7 @@ var valid = require("../utilities").isValidChannelName;
var fs = require("fs");
var path = require("path");
var Logger = require("../logger");
var tables = require("./tables");
var blackHole = function () { };
@ -10,47 +11,13 @@ function dropTable(name, callback) {
db.query("DROP TABLE `" + name + "`", callback);
}
function createRanksTable(name, callback) {
db.query("CREATE TABLE `chan_" + name + "_ranks` (" +
"`name` VARCHAR(20) NOT NULL," +
"`rank` INT NOT NULL," +
"PRIMARY KEY (`name`)) " +
"CHARACTER SET utf8", callback);
}
function createLibraryTable(name, callback) {
db.query("CREATE TABLE `chan_" + name + "_library` (" +
"`id` VARCHAR(255) NOT NULL," +
"`title` VARCHAR(255) NOT NULL," +
"`seconds` INT NOT NULL," +
"`type` VARCHAR(2) NOT NULL," +
"PRIMARY KEY (`id`))" +
"CHARACTER SET utf8", callback);
}
function createBansTable(name, callback) {
db.query("CREATE TABLE `chan_" + name + "_bans` (" +
"`id` INT NOT NULL AUTO_INCREMENT," +
"`ip` VARCHAR(39) NOT NULL," +
"`name` VARCHAR(20) NOT NULL," +
"`bannedby` VARCHAR(20) NOT NULL," +
"`reason` VARCHAR(255) NOT NULL," +
"PRIMARY KEY (`id`), UNIQUE (`name`, `ip`))" +
"CHARACTER SET utf8", callback);
}
function initTables(name, owner, callback) {
if (!valid(name)) {
callback("Invalid channel name", null);
return;
}
createRanksTable(name, function (err) {
if (err) {
callback(err, null);
return;
}
tables.createChannelTables(name, db.query.bind(db), function (err) {
db.users.getGlobalRank(owner, function (err, rank) {
if (err) {
callback(err, null);
@ -62,19 +29,6 @@ function initTables(name, owner, callback) {
module.exports.setRank(name, owner, rank, function (err) {
if (err) {
dropTable("chan_" + name + "_ranks");
callback(err, null);
return;
}
createLibraryTable(name, function (err) {
if (err) {
dropTable("chan_" + name + "_ranks");
callback(err, null);
return;
}
createBansTable(name, function (err) {
if (err) {
dropTable("chan_" + name + "_ranks");
dropTable("chan_" + name + "_library");
callback(err, null);
@ -85,22 +39,10 @@ function initTables(name, owner, callback) {
});
});
});
});
});
}
module.exports = {
/**
* Initialize the channels table
*/
init: function () {
db.query("CREATE TABLE IF NOT EXISTS `channels` (" +
"`id` INT NOT NULL AUTO_INCREMENT," +
"`name` VARCHAR(30) NOT NULL," +
"`owner` VARCHAR(20) NOT NULL," +
"`time` BIGINT NOT NULL," +
"PRIMARY KEY (`id`), INDEX(`name`), INDEX(`owner`))" +
"CHARACTER SET utf8");
},
/**

147
lib/database/tables.js Normal file
View File

@ -0,0 +1,147 @@
const TBL_USERS = "" +
"CREATE TABLE IF NOT EXISTS `users` (" +
"`id` INT NOT NULL AUTO_INCREMENT," +
"`name` VARCHAR(20) NOT NULL," +
"`password` VARCHAR(64) NOT NULL," +
"`global_rank` INT NOT NULL," +
"`email` VARCHAR(255) NOT NULL," +
"`profile` TEXT NOT NULL," +
"`ip` VARCHAR(39) NOT NULL," + "`time` BIGINT NOT NULL," +
"PRIMARY KEY(`id`)," +
"UNIQUE(`name`)) " +
"CHARACTER SET utf8";
const TBL_CHANNELS = "" +
"CREATE TABLE IF NOT EXISTS `channels` (" +
"`id` INT NOT NULL AUTO_INCREMENT," +
"`name` VARCHAR(30) NOT NULL," +
"`owner` VARCHAR(20) NOT NULL," +
"`time` BIGINT NOT NULL," +
"PRIMARY KEY (`id`), UNIQUE(`name`), INDEX(`owner`))" +
"CHARACTER SET utf8";
const TBL_GLOBAL_BANS = "" +
"CREATE TABLE IF NOT EXISTS `global_bans` (" +
"`ip` VARCHAR(39) NOT NULL," +
"`reason` VARCHAR(255) NOT NULL," +
"PRIMARY KEY (`ip`)) " +
"CHARACTER SET utf8";
const TBL_PASSWORD_RESET = "" +
"CREATE TABLE IF NOT EXISTS `password_reset` (" +
"`ip` VARCHAR(39) NOT NULL," +
"`name` VARCHAR(20) NOT NULL," +
"`hash` VARCHAR(64) NOT NULL," +
"`email` VARCHAR(255) NOT NULL," +
"`expire` BIGINT NOT NULL," +
"PRIMARY KEY (`name`))" +
"CHARACTER SET utf8";
const TBL_USER_PLAYLISTS = "" +
"CREATE TABLE IF NOT EXISTS `user_playlists` (" +
"`user` VARCHAR(20) NOT NULL," +
"`name` VARCHAR(255) NOT NULL," +
"`contents` MEDIUMTEXT NOT NULL," +
"`count` INT NOT NULL," +
"`duration` INT NOT NULL," +
"PRIMARY KEY (`user`, `name`))" +
"CHARACTER SET utf8";
const TBL_ALIASES = "" +
"CREATE TABLE IF NOT EXISTS `aliases` (" +
"`visit_id` INT NOT NULL AUTO_INCREMENT," +
"`ip` VARCHAR(39) NOT NULL," +
"`name` VARCHAR(20) NOT NULL," +
"`time` BIGINT NOT NULL," +
"PRIMARY KEY (`visit_id`), INDEX (`ip`)" +
")";
const TBL_STATS = "" +
"CREATE TABLE IF NOT EXISTS `stats` (" +
"`time` BIGINT NOT NULL," +
"`usercount` INT NOT NULL," +
"`chancount` INT NOT NULL," +
"`mem` INT NOT NULL," +
"PRIMARY KEY (`time`))" +
"CHARACTER SET utf8";
const TBL_META = "" +
"CREATE TABLE IF NOT EXISTS `meta` (" +
"`key` VARCHAR(255) NOT NULL," +
"`value` TEXT NOT NULL," +
"PRIMARY KEY (`key`))" +
"CHARACTER SET utf8";
module.exports.init = function (queryfn, cb) {
var tables = {
users: TBL_USERS,
channels: TBL_CHANNELS,
global_bans: TBL_GLOBAL_BANS,
password_reset: TBL_PASSWORD_RESET,
user_playlists: TBL_USER_PLAYLISTS,
aliases: TBL_ALIASES,
stats: TBL_STATS,
meta: TBL_META
};
var AsyncQueue = require("../asyncqueue");
var aq = new AsyncQueue();
var hasError = false;
Object.keys(tables).forEach(function (tbl) {
aq.queue(function (lock) {
queryfn(tables[tbl], function (err) {
if (err) {
console.log(err);
hasError = true;
}
lock.release();
});
});
});
aq.queue(function (lock) {
lock.release();
cb(hasError);
});
};
module.exports.createChannelTables = function (name, queryfn, cb) {
var createRanksTable = function () {
queryfn("CREATE TABLE `chan_" + name + "_ranks` (" +
"`name` VARCHAR(20) NOT NULL," +
"`rank` INT NOT NULL," +
"PRIMARY KEY (`name`)) " +
"CHARACTER SET utf8", createLibraryTable);
};
var createLibraryTable = function (err) {
if (err) {
cb(err);
return;
}
queryfn("CREATE TABLE `chan_" + name + "_library` (" +
"`id` VARCHAR(255) NOT NULL," +
"`title` VARCHAR(255) NOT NULL," +
"`seconds` INT NOT NULL," +
"`type` VARCHAR(2) NOT NULL," +
"PRIMARY KEY (`id`))" +
"CHARACTER SET utf8", createBansTable);
};
var createBansTable = function (err) {
if (err) {
cb(err);
return;
}
queryfn("CREATE TABLE `chan_" + name + "_bans` (" +
"`id` INT NOT NULL AUTO_INCREMENT," +
"`ip` VARCHAR(39) NOT NULL," +
"`name` VARCHAR(20) NOT NULL," +
"`bannedby` VARCHAR(20) NOT NULL," +
"`reason` VARCHAR(255) NOT NULL," +
"PRIMARY KEY (`id`), UNIQUE (`name`, `ip`))" +
"CHARACTER SET utf8", cb);
};
createRanksTable();
};