moon
/
sync
mirror of https://github.com/calzoneman/sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix UTF8 issues - run 'node update.js 2013-08-20-utf8fix' to correct your database

This commit is contained in:
calzoneman 2013-08-20 19:50:49 -05:00
parent e830ba2d41
commit ffddc434f2
2 changed files with 129 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
database.js
View File

@ -20,7 +20,8 @@ var Database = function (cfg) {
Logger.errlog.log("! DB connection failed"); Logger.errlog.log("! DB connection failed");
return; return;
} }
conn.end();
}); });
this.global_ipbans = {}; this.global_ipbans = {};
@ -62,9 +63,7 @@ Database.prototype.query = function (query, sub, callback) {
} }
} }
}); });
} } function blackHole() {
function blackHole() {
} }
@ -77,7 +76,8 @@ Database.prototype.init = function () {
"`name` VARCHAR(255) NOT NULL,", "`name` VARCHAR(255) NOT NULL,",
"`owner` VARCHAR(20) NOT NULL,", "`owner` VARCHAR(20) NOT NULL,",
"PRIMARY KEY(`id`))", "PRIMARY KEY(`id`))",
"ENGINE = MyISAM;"].join(""); "ENGINE = MyISAM ",
"CHARACTER SET utf8;"].join("");
self.query(query, function (err, res) { self.query(query, function (err, res) {
if(err) { if(err) {
@ -97,7 +97,8 @@ Database.prototype.init = function () {
"`profile_text` TEXT NOT NULL,", "`profile_text` TEXT NOT NULL,",
"`email` VARCHAR(255) NOT NULL,", "`email` VARCHAR(255) NOT NULL,",
"PRIMARY KEY (`id`))", "PRIMARY KEY (`id`))",
"ENGINE = MyISAM;"].join(""); "ENGINE = MyISAM ",
"CHARACTER SET utf8;"].join("");
self.query(query, function (err, res) { self.query(query, function (err, res) {
if(err) { if(err) {
@ -110,7 +111,8 @@ Database.prototype.init = function () {
"`ip` VARCHAR(15) NOT NULL,", "`ip` VARCHAR(15) NOT NULL,",
"`note` VARCHAR(255) NOT NULL,", "`note` VARCHAR(255) NOT NULL,",
"PRIMARY KEY (`ip`))", "PRIMARY KEY (`ip`))",
"ENGINE = MyISAM;"].join(""); "ENGINE = MyISAM ",
"CHARACTER SET utf8;"].join("");
self.query(query, function (err, res) { self.query(query, function (err, res) {
if(err) { if(err) {
@ -126,7 +128,8 @@ Database.prototype.init = function () {
"`email` VARCHAR(255) NOT NULL,", "`email` VARCHAR(255) NOT NULL,",
"`expire` BIGINT NOT NULL,", "`expire` BIGINT NOT NULL,",
"PRIMARY KEY (`name`))", "PRIMARY KEY (`name`))",
"ENGINE = MyISAM;"].join(""); "ENGINE = MyISAM ",
"CHARACTER SET utf8;"].join("");
self.query(query, function (err, res) { self.query(query, function (err, res) {
if(err) { if(err) {
@ -142,7 +145,8 @@ Database.prototype.init = function () {
"`count` INT NOT NULL,", "`count` INT NOT NULL,",
"`time` INT NOT NULL,", "`time` INT NOT NULL,",
"PRIMARY KEY (`user`, `name`))", "PRIMARY KEY (`user`, `name`))",
"ENGINE = MyISAM;"].join(""); "ENGINE = MyISAM ",
"CHARACTER SET utf8;"].join("");
self.query(query, function (err, res) { self.query(query, function (err, res) {
if(err) { if(err) {
@ -157,7 +161,8 @@ Database.prototype.init = function () {
"`name` VARCHAR(20) NOT NULL,", "`name` VARCHAR(20) NOT NULL,",
"`time` BIGINT NOT NULL,", "`time` BIGINT NOT NULL,",
"PRIMARY KEY (`visit_id`), INDEX (`ip`))", "PRIMARY KEY (`visit_id`), INDEX (`ip`))",
"ENGINE = MyISAM;"].join(""); "ENGINE = MyISAM ",
"CHARACTER SET utf8;"].join("");
self.query(query, function (err, res) { self.query(query, function (err, res) {
if(err) { if(err) {
@ -173,7 +178,8 @@ Database.prototype.init = function () {
"`args` TEXT NOT NULL,", "`args` TEXT NOT NULL,",
"`time` BIGINT NOT NULL,", "`time` BIGINT NOT NULL,",
"PRIMARY KEY (`ip`, `time`), INDEX (`action`))", "PRIMARY KEY (`ip`, `time`), INDEX (`action`))",
"ENGINE = MyISAM;"].join(""); "ENGINE = MyISAM ",
"CHARACTER SET utf8;"].join("");
self.query(query, function (err, res) { self.query(query, function (err, res) {
if(err) { if(err) {
@ -188,7 +194,8 @@ Database.prototype.init = function () {
"`chancount` INT NOT NULL,", "`chancount` INT NOT NULL,",
"`mem` INT NOT NULL,", "`mem` INT NOT NULL,",
"PRIMARY KEY (`time`))", "PRIMARY KEY (`time`))",
"ENGINE = MyISAM;"].join(""); "ENGINE = MyISAM ",
"CHARACTER SET utf8;"].join("");
self.query(query, function (err, res) { self.query(query, function (err, res) {
if(err) { if(err) {
@ -330,7 +337,8 @@ Database.prototype.registerChannel = function (name, owner, callback) {
"`seconds` INT NOT NULL,", "`seconds` INT NOT NULL,",
"`type` VARCHAR(2) NOT NULL,", "`type` VARCHAR(2) NOT NULL,",
"PRIMARY KEY (`id`))", "PRIMARY KEY (`id`))",
"ENGINE = MyISAM;"].join(""); "ENGINE = MyISAM ",
"CHARACTER SET utf8;"].join("");
self.query(query, function (err, res) { self.query(query, function (err, res) {
if(err) { if(err) {
callback(err, null); callback(err, null);
@ -342,7 +350,8 @@ Database.prototype.registerChannel = function (name, owner, callback) {
"`name` VARCHAR(32) NOT NULL,", "`name` VARCHAR(32) NOT NULL,",
"`rank` INT NOT NULL,", "`rank` INT NOT NULL,",
"UNIQUE (`name`))", "UNIQUE (`name`))",
"ENGINE = MyISAM;"].join(""); "ENGINE = MyISAM ",
"CHARACTER SET utf8;"].join("");
self.query(query, function (err, res) { self.query(query, function (err, res) {
if(err) { if(err) {
@ -356,7 +365,8 @@ Database.prototype.registerChannel = function (name, owner, callback) {
"`name` VARCHAR(32) NOT NULL,", "`name` VARCHAR(32) NOT NULL,",
"`banner` VARCHAR(32) NOT NULL,", "`banner` VARCHAR(32) NOT NULL,",
"PRIMARY KEY (`ip`))", "PRIMARY KEY (`ip`))",
"ENGINE = MyISAM;"].join(""); "ENGINE = MyISAM ",
"CHARACTER SET utf8;"].join("");
self.query(query, function (err, res) { self.query(query, function (err, res) {
if(err) { if(err) {
@ -558,7 +568,8 @@ Database.prototype.addToLibrary = function (channame, media, callback) {
return; return;
} }
var query = "INSERT INTO `chan_" + channame + "_library` " + // use INSERT IGNORE to prevent errors from adding duplicates
var query = "INSERT IGNORE INTO `chan_" + channame + "_library` " +
"(id, title, seconds, type) " + "(id, title, seconds, type) " +
"VALUES (?, ?, ?, ?)"; "VALUES (?, ?, ?, ?)";
var params = [ var params = [

127
update.js
View File

@ -1,35 +1,112 @@
var Config = require("./config.js"); var Config = require("./config.js");
var Database = require("./database.js"); var Database = require("./database.js");
var updates = {
"2013-08-20-utf8fix": fixDBUnicode
};
var x = {}; var x = {};
Config.load(x, "cfg.json", function () { Config.load(x, "cfg.json", function () {
Database.setup(x.cfg); var db = new Database(x.cfg);
Database.init();
var query; var u = process.argv[2];
var db = Database.getConnection(); if(!(u in updates)) {
console.log("Usage: node update.js <update>");
console.log("Available updates: ");
query = "SELECT * FROM channels WHERE 1"; for(var k in updates) {
var res = db.querySync(query); console.log(" " + k);
if(!res) { }
console.log(db); process.exit(0);
console.log("Update failed!");
return;
} }
var channels = res.fetchAllSync(); var fn = updates[u];
channels.forEach(function(chan) { fn(db);
chan = chan.name; });
query = "UPDATE `chan_" + chan + "_library` SET title=CONCAT(" +
"SUBSTRING(title FROM 0 FOR 97), '...') WHERE " + function fixDBUnicode(db) {
"LENGTH(title) > 100"; db.query("SHOW TABLES", function (err, res) {
console.log(query); if(err) {
res = db.querySync(query); console.log(err);
if(!res) {
console.log(db);
console.log("failed to fix "+chan);
return; return;
} }
var libs = [];
var waiting = res.length;
res.forEach(function (r) {
var k = Object.keys(r)[0];
if(r[k].match(/^chan_[\w-_]{1,30}_library$/)) {
libs.push(r[k]);
waiting--;
return;
} else if(r[k] == "user_playlists") {
waiting--;
return;
}
db.query("ALTER TABLE `" + r[k] + "` CONVERT TO CHARACTER SET utf8", function (err, _) {
waiting--;
if(err)
console.log("FAIL: " + r[k]);
else
console.log("Fixed " + r[k]);
});
});
var s1int = setInterval(function () {
if(waiting == 0) {
clearInterval(s1int);
waiting = libs.length + 1;
libs.forEach(function (lib) {
db.query("ALTER TABLE `"+lib+"` MODIFY title BLOB", function (err, _) {
if(err) {
console.log(err);
waiting--;
return;
}
db.query("ALTER TABLE `"+lib+"` CHARACTER SET utf8", function (err, _) {
if(err) {
console.log(err);
waiting--;
return;
}
db.query("ALTER TABLE `"+lib+"` MODIFY title VARCHAR(255) CHARACTER SET utf8", function (err, _) {
waiting--;
if(err) {
console.log(err);
return;
}
console.log("Fixed user_playlists");
});
});
});
});
db.query("ALTER TABLE user_playlists MODIFY contents MEDIUMBLOB", function (err, _) {
if(err) {
console.log(err);
waiting--;
return;
}
db.query("ALTER TABLE user_playlists CHARACTER SET utf8", function (err, _) {
if(err) {
console.log(err);
waiting--;
return;
}
db.query("ALTER TABLE user_playlists MODIFY contents MEDIUMTEXT CHARACTER SET utf8", function (err, _) {
waiting--;
if(err) {
console.log(err);
return;
}
console.log("Fixed " + lib);
});
});
});
setInterval(function () {
if(waiting == 0) {
console.log("Done");
process.exit(0);
}
}, 1000);
}
}, 1000);
}); });
db.closeSync(); }
});