Start refactoring channel storage

This commit is contained in:
calzoneman 2015-09-20 23:17:06 -07:00
parent f95f3dc89b
commit 5ec9c2b029
5 changed files with 128 additions and 93 deletions

View File

@ -0,0 +1,11 @@
import { FileStore } from './filestore';
var CHANNEL_STORE = new FileStore();
export function load(channelName) {
return CHANNEL_STORE.load(channelName);
}
export function save(channelName, data) {
return CHANNEL_STORE.save(channelName, data);
}

View File

@ -0,0 +1,46 @@
import * as Promise from 'bluebird';
import { stat } from 'fs';
import * as fs from 'graceful-fs';
import path from 'path';
const readFileAsync = Promise.promisify(fs.readFile);
const writeFileAsync = Promise.promisify(fs.writeFile);
const statAsync = Promise.promisify(stat);
const SIZE_LIMIT = 1048576;
const CHANDUMP_DIR = path.resolve(__dirname, '..', '..', 'chandump');
export class FileStore {
filenameForChannel(channelName) {
return path.join(CHANDUMP_DIR, channelName);
}
load(channelName) {
const filename = this.filenameForChannel(channelName);
return statAsync(filename).then(stats => {
if (stats.size > SIZE_LIMIT) {
throw new Error('Channel state file is too large: ' + stats.size);
} else {
return readFileAsync(filename);
}
}).then(fileContents => {
try {
return JSON.parse(fileContents);
} catch (e) {
throw new Error('Channel state file is not valid JSON: ' + e);
}
});
}
save(channelName, data) {
const filename = this.filenameForChannel(channelName);
const fileContents = new Buffer(JSON.stringify(data), 'utf8');
if (fileContents.length > SIZE_LIMIT) {
let error = new Error('Channel state size is too large');
error.limit = SIZE_LIMIT;
error.size = fileContents.length;
return Promise.reject(error);
}
return writeFileAsync(filename, fileContents);
}
}

View File

@ -8,6 +8,8 @@ var fs = require("graceful-fs");
var path = require("path");
var sio = require("socket.io");
var db = require("../database");
var ChannelStore = require("../channel-storage/channelstore");
var Promise = require("bluebird");
const SIZE_LIMIT = 1048576;
@ -150,17 +152,15 @@ Channel.prototype.getDiskSize = function (cb) {
};
Channel.prototype.loadState = function () {
var self = this;
var file = path.join(__dirname, "..", "..", "chandump", self.uniqueName);
/* Don't load from disk if not registered */
if (!self.is(Flags.C_REGISTERED)) {
self.modules.permissions.loadUnregistered();
self.setFlag(Flags.C_READY);
if (!this.is(Flags.C_REGISTERED)) {
this.modules.permissions.loadUnregistered();
this.setFlag(Flags.C_READY);
return;
}
var errorLoad = function (msg) {
const self = this;
function errorLoad(msg) {
if (self.modules.customization) {
self.modules.customization.load({
motd: msg
@ -168,99 +168,73 @@ Channel.prototype.loadState = function () {
}
self.setFlag(Flags.C_READY | Flags.C_ERROR);
};
}
fs.stat(file, function (err, stats) {
if (!err) {
var mb = stats.size / 1048576;
mb = Math.floor(mb * 100) / 100;
if (mb > SIZE_LIMIT / 1048576) {
Logger.errlog.log("Large chandump detected: " + self.uniqueName +
" (" + mb + " MiB)");
var msg = "This channel's state size has exceeded the memory limit " +
"enforced by this server. Please contact an administrator " +
"for assistance.";
errorLoad(msg);
return;
}
}
continueLoad();
});
var continueLoad = function () {
fs.readFile(file, function (err, data) {
if (err) {
/* ENOENT means the file didn't exist. This is normal for new channels */
if (err.code === "ENOENT") {
self.setFlag(Flags.C_READY);
Object.keys(self.modules).forEach(function (m) {
self.modules[m].load({});
});
} else {
Logger.errlog.log("Failed to open channel dump " + self.uniqueName);
Logger.errlog.log(err);
errorLoad("Unknown error occurred when loading channel state. " +
"Contact an administrator for assistance.");
}
return;
}
self.logger.log("[init] Loading channel state from disk");
ChannelStore.load(this.uniqueName).then(data => {
Object.keys(this.modules).forEach(m => {
try {
data = JSON.parse(data);
Object.keys(self.modules).forEach(function (m) {
self.modules[m].load(data);
});
self.setFlag(Flags.C_READY);
this.modules[m].load(data);
} catch (e) {
Logger.errlog.log("Channel dump for " + self.uniqueName + " is not " +
"valid");
Logger.errlog.log(e);
errorLoad("Unknown error occurred when loading channel state. Contact " +
"an administrator for assistance.");
Logger.errlog.log("Failed to load module " + m + " for channel " +
this.uniqueName);
}
});
};
this.setFlag(Flags.C_READY);
}).catch(err => {
if (err.code === 'ENOENT') {
Object.keys(this.modules).forEach(m => {
this.modules[m].load({});
});
this.setFlag(Flags.C_READY);
return;
}
let message;
if (/Channel state file is too large/.test(err.message)) {
message = "This channel's state size has exceeded the memory limit " +
"enforced by this server. Please contact an administrator " +
"for assistance.";
} else {
message = "An error occurred when loading this channel's data from " +
"disk. Please contact an administrator for assistance. " +
`The error was: ${err}`;
}
Logger.errlog.log(err.stack);
errorLoad(message);
});
};
Channel.prototype.saveState = function () {
var self = this;
var file = path.join(__dirname, "..", "..", "chandump", self.uniqueName);
/**
* Don't overwrite saved state data if the current state is dirty,
* or if this channel is unregistered
*/
if (self.is(Flags.C_ERROR) || !self.is(Flags.C_REGISTERED)) {
return;
if (!this.is(Flags.C_REGISTERED)) {
return Promise.resolve();
}
self.logger.log("[init] Saving channel state to disk");
var data = {};
Object.keys(this.modules).forEach(function (m) {
self.modules[m].save(data);
if (this.is(Flags.C_ERROR)) {
return Promise.reject(new Error(`Channel is in error state`));
}
this.logger.log("[init] Saving channel state to disk");
const data = {};
Object.keys(this.modules).forEach(m => {
this.modules[m].save(data);
});
var json = JSON.stringify(data);
/**
* Synchronous on purpose.
* When the server is shutting down, saveState() is called on all channels and
* then the process terminates. Async writeFile causes a race condition that wipes
* channels.
*/
var err = fs.writeFileSync(file, json);
// Check for large chandump and warn moderators/admins
self.getDiskSize(function (err, size) {
if (!err && size > SIZE_LIMIT && self.users) {
self.users.forEach(function (u) {
return ChannelStore.save(this.uniqueName, data).catch(err => {
if (/Channel state size is too large/.test(err.message)) {
this.users.forEach(u => {
if (u.account.effectiveRank >= 2) {
u.socket.emit("warnLargeChandump", {
limit: SIZE_LIMIT,
actual: size
limit: err.limit,
actual: err.size
});
}
});
Logger.errlog.log(`Not saving ${this.uniqueName} because it exceeds ` +
"the size limit");
} else {
Logger.errlog.log(`Failed to save ${this.uniqueName}: ${err.stack}`);
}
});
};

View File

@ -1,6 +1,7 @@
const VERSION = require("../package.json").version;
var singleton = null;
var Config = require("./config");
var Promise = require("bluebird");
module.exports = {
init: function () {
@ -226,13 +227,15 @@ Server.prototype.announce = function (data) {
Server.prototype.shutdown = function () {
Logger.syslog.log("Unloading channels");
for (var i = 0; i < this.channels.length; i++) {
if (this.channels[i].is(Flags.C_REGISTERED)) {
Logger.syslog.log("Saving /r/" + this.channels[i].name);
this.channels[i].saveState();
}
}
Logger.syslog.log("Goodbye");
process.exit(0);
Promise.map(this.channels, channel => {
return channel.saveState().tap(() => {
Logger.syslog.log(`Saved /r/${channel.name}`);
}).catch(err => {
Logger.errlog.log(`Failed to save /r/${channel.name}: ${err.stack}`);
});
}).then(() => {
Logger.syslog.log("Goodbye");
process.exit(0);
});
};

View File

@ -1072,8 +1072,9 @@ Callbacks = {
errDialog("This channel currently exceeds the maximum size of " +
toHumanReadable(data.limit) + " (channel size is " +
toHumanReadable(data.actual) + "). Please reduce the size by removing " +
"unneeded playlist items, filters, and/or emotes or else the channel will " +
"be unable to load the next time it is reloaded").attr("id", "chandumptoobig");
"unneeded playlist items, filters, and/or emotes. Changes to the channel " +
"will not be saved until the size is reduced to under the limit.")
.attr("id", "chandumptoobig");
}
}