From 9deff9bdb1fac4ac457f443e7ce9a039aeb2d5bf Mon Sep 17 00:00:00 2001 From: Calvin Montgomery Date: Sun, 14 Dec 2014 21:33:48 -0500 Subject: [PATCH] Change charset for certain fields to utf8mb4 The underlying cause of #419 is the default utf8 collation in MySQL/MariaDB, which only supports the base plane of Unicode (\u0000-\uffff). By changing the collation to utf8mb4_general_ci, stuff like ban reasons and profile text may have emoji and other non-base-plane Unicode. The charset for playlist titles is NOT changed, and non-base-plane characters are replaced by question marks. This is because switching to utf8mb4 would make the primary key too long. --- lib/channel/playlist.js | 2 ++ lib/database.js | 3 ++- lib/database/tables.js | 8 ++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/channel/playlist.js b/lib/channel/playlist.js index ae9c12f1..4d8ffff0 100644 --- a/lib/channel/playlist.js +++ b/lib/channel/playlist.js @@ -1243,6 +1243,8 @@ PlaylistModule.prototype.handleClonePlaylist = function (user, data) { }); } + data.name = data.name.replace(/[^\u0000-\uffff]/g, "?"); + var pl = this.items.toArray(); var self = this; db.saveUserPlaylist(pl, user.getName(), data.name, function (err) { diff --git a/lib/database.js b/lib/database.js index 1faecbac..b77b68f8 100644 --- a/lib/database.js +++ b/lib/database.js @@ -17,7 +17,8 @@ module.exports.init = function () { user: Config.get("mysql.user"), password: Config.get("mysql.password"), database: Config.get("mysql.database"), - multipleStatements: true + multipleStatements: true, + charset: "UTF8MB4_GENERAL_CI" // Needed for emoji and other non-BMP unicode }); // Test the connection diff --git a/lib/database/tables.js b/lib/database/tables.js index 4fbd7a09..0b4cb841 100644 --- a/lib/database/tables.js +++ b/lib/database/tables.js @@ -5,7 +5,7 @@ const TBL_USERS = "" + "`password` VARCHAR(64) NOT NULL," + "`global_rank` INT NOT NULL," + "`email` VARCHAR(255) NOT NULL," + - "`profile` TEXT NOT NULL," + + "`profile` TEXT CHARACTER SET utf8mb4 NOT NULL," + "`ip` VARCHAR(39) NOT NULL," + "`time` BIGINT NOT NULL," + "PRIMARY KEY(`id`)," + "UNIQUE(`name`)) " + @@ -23,7 +23,7 @@ const TBL_CHANNELS = "" + const TBL_GLOBAL_BANS = "" + "CREATE TABLE IF NOT EXISTS `global_bans` (" + "`ip` VARCHAR(39) NOT NULL," + - "`reason` VARCHAR(255) NOT NULL," + + "`reason` VARCHAR(255) CHARACTER SET utf8mb4 NOT NULL," + "PRIMARY KEY (`ip`)) " + "CHARACTER SET utf8"; @@ -75,7 +75,7 @@ const TBL_META = "" + const TBL_LIBRARIES = "" + "CREATE TABLE IF NOT EXISTS `channel_libraries` (" + "`id` VARCHAR(255) NOT NULL," + - "`title` VARCHAR(255) NOT NULL," + + "`title` VARCHAR(255) CHARACTER SET utf8mb4 NOT NULL," + "`seconds` INT NOT NULL," + "`type` VARCHAR(2) NOT NULL," + "`meta` TEXT NOT NULL," + @@ -97,7 +97,7 @@ const TBL_BANS = "" + "`ip` VARCHAR(39) NOT NULL," + "`name` VARCHAR(20) NOT NULL," + "`bannedby` VARCHAR(20) NOT NULL," + - "`reason` VARCHAR(255) NOT NULL," + + "`reason` VARCHAR(255) CHARACTER SET utf8mb4 NOT NULL," + "`channel` VARCHAR(30) NOT NULL," + "PRIMARY KEY (`id`, `channel`), UNIQUE (`name`, `ip`, `channel`), " + "INDEX (`ip`, `channel`), INDEX (`name`, `channel`)" +